tablefacts 0.1.0

package/src/tripadvisor/links.mjs

@@ -0,0 +1,78 @@
+// The pure parts of photos.mjs: the command line, the restaurant link and the photo URLs.
+// Split out so they can be tested without starting a browser.
+
+import { usageError } from '../lib/errors.mjs'
+
+export function parseArgs(argv) {
+  const opts = { links: [], dryRun: false, debug: false, max: Infinity }
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i]
+    if (a === '--out') opts.out = argv[++i]
+    else if (a === '--cdp') opts.cdp = argv[++i]
+    else if (a === '--edge-dir') opts.edgeDir = argv[++i]
+    else if (a === '--max') {
+      opts.max = Number(argv[++i])
+      if (!(opts.max > 0)) throw usageError('--max needs a number above 0')
+    } else if (a === '--dry-run') opts.dryRun = true
+    else if (a === '--debug') opts.debug = true
+    else if (a === '--help' || a === '-h') opts.help = true
+    else if (a.startsWith('--')) throw usageError(`Unknown option ${a}`)
+    else opts.links.push(a)
+  }
+  return opts
+}
+
+// A restaurant page looks like /Restaurant_Review-g297478-d1234567-Reviews-Name-City.html on
+// any country domain (tripadvisor.com, .co, .es...). The query and hash are dropped, so the same
+// restaurant gives one URL.
+export function normalizeRestaurant(raw) {
+  let url
+  try {
+    url = new URL(raw.trim())
+  } catch {
+    return null
+  }
+  if (!/(^|\.)tripadvisor\.[a-z.]+$/.test(url.hostname)) return null
+  const m = url.pathname.match(/\/Restaurant_Review-g\d+-(d\d+)[^/]*\.html$/)
+  if (!m) return null
+  return { id: m[1], url: `${url.origin}${url.pathname}` }
+}
+
+const MEDIA = /https?:\/\/(?:dynamic-media-cdn|media-cdn)\.tripadvisor\.com\/media\/photo-([a-z])\/(?:\d{3,}\/)?([\w-]+(?:\/[\w-]+){2,5}\/[^\s"'\\)?<>]+?\.(?:jpe?g|png|webp))/gi
+
+// Page HTML often carries URLs inside JSON, with slashes escaped as \/ or \u002F.
+function unescape(text) {
+  return text.replace(/\\u002F/gi, '/').replace(/\\\//g, '/').replace(/&amp;/g, '&')
+}
+
+// Every TripAdvisor photo URL in some text, once per photo (the same photo appears in several
+// sizes: photo-s, photo-l, photo-o...). Reviewer avatars, maps and logos are left out.
+// Returns [{ key, size, url }] in the order first seen; `key` is the path after the size.
+export function findPhotos(text) {
+  const found = new Map()
+  for (const m of unescape(text).matchAll(MEDIA)) {
+    const [url, size, key] = m
+    if (/avatar|\/map|logo/i.test(key)) continue
+    if (!found.has(key)) found.set(key, { key, size, url })
+  }
+  return [...found.values()]
+}
+
+// Candidate URLs for one photo, biggest first: the original, then the next sizes, then the size the
+// page itself showed. Not every size exists for every photo, so the caller tries them in turn.
+export function sizeCandidates(photo) {
+  const base = 'https://dynamic-media-cdn.tripadvisor.com/media'
+  const urls = ['o', 'w', 'l'].map((size) => `${base}/photo-${size}/${photo.key}`)
+  const own = photo.url.replace(/\?.*$/, '')
+  if (!urls.includes(own)) urls.push(own)
+  return urls
+}
+
+// '1a/2b/3c/4d/name.jpg' -> '1a2b3c4d-name.jpg': unique per photo and stable between runs.
+export function fileName(photo) {
+  const parts = photo.key.split('/')
+  const name = parts.pop()
+  const ext = (name.match(/\.(jpe?g|png|webp)$/i)?.[1] ?? 'jpg').toLowerCase().replace('jpeg', 'jpg')
+  const stem = name.replace(/\.\w+$/, '')
+  return `${parts.join('')}-${stem}.${ext}`.replace(/[^\w.-]/g, '_')
+}

package/src/tripadvisor/photos.mjs

@@ -0,0 +1,55 @@
+// Downloads the photos of a restaurant's TripAdvisor page, in the user's own Edge window.
+// Usage: tablefacts photos tripadvisor --out <folder> [--cdp <url>] <restaurant link>...
+// The work is in index.mjs (downloadTripadvisor); this file is only the command line.
+import { TablefactsError, cliMessage, exitCodeFor } from '../lib/errors.mjs'
+import { consoleLog } from '../lib/log.mjs'
+import { downloadTripadvisor } from './index.mjs'
+import { parseArgs } from './links.mjs'
+
+const HELP = `Download the photos of a restaurant's TripAdvisor page.
+
+  tablefacts photos tripadvisor --out <folder> [options] <restaurant link>...
+
+  --out <folder>   where the images are saved (required, created if missing)
+  --cdp <url>      attach to Edge on this debugging address (default http://localhost:9222). If nothing
+                   is listening there the script starts Edge itself
+  --edge-dir <dir> profile folder of that Edge (default C:\\ig-edge, shared with photos instagram)
+  --max <n>        stop after n photos per restaurant (default: all it can reach)
+  --dry-run        print the photo URLs found, save nothing
+  --debug          save the page HTML into <folder>/_debug, to fix the selectors
+  --help           this text
+
+The link is the restaurant's page, like https://www.tripadvisor.com/Restaurant_Review-g297478-d123-Reviews-Name.html
+TripAdvisor shows a "verify you are human" check to automated browsers: if it appears in the window,
+tick it yourself. This script waits for it and never tries to get around it.
+
+Only download photos the restaurant owns or has allowed you to use. Most photos on TripAdvisor were
+uploaded by guests, who keep the rights to them.`
+
+const FLAGS = {
+  out: '--out <folder>',
+  links: 'the arguments',
+  cdp: '--cdp',
+  debug: '--debug',
+}
+
+let summary
+try {
+  const opts = parseArgs(process.argv.slice(2))
+  if (opts.help) {
+    console.log(HELP)
+    process.exit(0)
+  }
+  if (!opts.out) {
+    console.error('Missing --out <folder>.\n\n' + HELP)
+    process.exit(2)
+  }
+  summary = await downloadTripadvisor({ ...opts, log: consoleLog })
+} catch (err) {
+  if (!(err instanceof TablefactsError)) throw err
+  console.error(cliMessage(err, FLAGS))
+  process.exit(exitCodeFor(err))
+}
+console.log(`\nSaved ${summary.saved}, already there ${summary.skipped}, failed ${summary.failed.length}.`)
+for (const f of summary.failed) console.log(`  ${f.item}: ${cliMessage({ message: f.reason }, FLAGS)}`)
+process.exit(summary.failed.length ? 1 : 0)

package/types/index.d.mts

@@ -0,0 +1,65 @@
+/** @typedef {import('./lib/types.mjs').Log} Log */
+/** @typedef {import('./lib/types.mjs').Env} Env */
+/** @typedef {import('./lib/types.mjs').ResearchOptions} ResearchOptions */
+/** @typedef {import('./lib/types.mjs').ResearchResult} ResearchResult */
+/** @typedef {import('./lib/types.mjs').ResearchPhoto} ResearchPhoto */
+/** @typedef {import('./lib/types.mjs').ResearchProfile} ResearchProfile */
+/** @typedef {import('./lib/types.mjs').InstagramOptions} InstagramOptions */
+/** @typedef {import('./lib/types.mjs').TripadvisorOptions} TripadvisorOptions */
+/** @typedef {import('./lib/types.mjs').PhotoSummary} PhotoSummary */
+/** @typedef {import('./lib/types.mjs').PhotoFailure} PhotoFailure */
+/** @typedef {import('./lib/types.mjs').Menu} Menu */
+/** @typedef {import('./lib/types.mjs').MenuCategory} MenuCategory */
+/** @typedef {import('./lib/types.mjs').MenuSection} MenuSection */
+/** @typedef {import('./lib/types.mjs').MenuProduct} MenuProduct */
+/** @typedef {import('./lib/types.mjs').MenuTotals} MenuTotals */
+/** @typedef {import('./lib/types.mjs').MenuImage} MenuImage */
+/** @typedef {import('./lib/types.mjs').MenuSourceConfig} MenuSourceConfig */
+/** @typedef {import('./lib/types.mjs').CluviConfig} CluviConfig */
+/** @typedef {import('./lib/types.mjs').RawConfig} RawConfig */
+/** @typedef {import('./lib/types.mjs').ImportOptions} ImportOptions */
+/** @typedef {import('./lib/types.mjs').ImportMenuOptions} ImportMenuOptions */
+/** @typedef {import('./lib/types.mjs').ImportCluviOptions} ImportCluviOptions */
+/** @typedef {import('./lib/types.mjs').ImportImageMenuOptions} ImportImageMenuOptions */
+/** @typedef {import('./lib/types.mjs').ListMenuImagesOptions} ListMenuImagesOptions */
+/** @typedef {import('./lib/types.mjs').ImportResult} ImportResult */
+/** @typedef {import('./lib/types.mjs').PriceFormat} PriceFormat */
+/** @typedef {import('./lib/types.mjs').VisionProvider} VisionProvider */
+/** @typedef {import('./lib/types.mjs').LoadEnvOptions} LoadEnvOptions */
+/** @typedef {import('./lib/types.mjs').TablefactsErrorCode} TablefactsErrorCode */
+export type Log = import('./lib/types.mjs').Log;
+export type Env = import('./lib/types.mjs').Env;
+export type ResearchOptions = import('./lib/types.mjs').ResearchOptions;
+export type ResearchResult = import('./lib/types.mjs').ResearchResult;
+export type ResearchPhoto = import('./lib/types.mjs').ResearchPhoto;
+export type ResearchProfile = import('./lib/types.mjs').ResearchProfile;
+export type InstagramOptions = import('./lib/types.mjs').InstagramOptions;
+export type TripadvisorOptions = import('./lib/types.mjs').TripadvisorOptions;
+export type PhotoSummary = import('./lib/types.mjs').PhotoSummary;
+export type PhotoFailure = import('./lib/types.mjs').PhotoFailure;
+export type Menu = import('./lib/types.mjs').Menu;
+export type MenuCategory = import('./lib/types.mjs').MenuCategory;
+export type MenuSection = import('./lib/types.mjs').MenuSection;
+export type MenuProduct = import('./lib/types.mjs').MenuProduct;
+export type MenuTotals = import('./lib/types.mjs').MenuTotals;
+export type MenuImage = import('./lib/types.mjs').MenuImage;
+export type MenuSourceConfig = import('./lib/types.mjs').MenuSourceConfig;
+export type CluviConfig = import('./lib/types.mjs').CluviConfig;
+export type RawConfig = import('./lib/types.mjs').RawConfig;
+export type ImportOptions = import('./lib/types.mjs').ImportOptions;
+export type ImportMenuOptions = import('./lib/types.mjs').ImportMenuOptions;
+export type ImportCluviOptions = import('./lib/types.mjs').ImportCluviOptions;
+export type ImportImageMenuOptions = import('./lib/types.mjs').ImportImageMenuOptions;
+export type ListMenuImagesOptions = import('./lib/types.mjs').ListMenuImagesOptions;
+export type ImportResult = import('./lib/types.mjs').ImportResult;
+export type PriceFormat = import('./lib/types.mjs').PriceFormat;
+export type VisionProvider = import('./lib/types.mjs').VisionProvider;
+export type LoadEnvOptions = import('./lib/types.mjs').LoadEnvOptions;
+export type TablefactsErrorCode = import('./lib/types.mjs').TablefactsErrorCode;
+export { research } from './research/index.mjs';
+export { downloadInstagram } from './instagram/index.mjs';
+export { downloadTripadvisor } from './tripadvisor/index.mjs';
+export { importMenu, importCluvi, importImageMenu, listMenuImages, validateMenu, countMenu, normalizePages, parsePrice, providers, defaultProvider, } from './menu/index.mjs';
+export { projectRoot, workDir, workDirIn, envFiles } from './lib/project.mjs';
+export { TablefactsError } from './lib/errors.mjs';
+export { loadEnv } from './lib/env.mjs';

package/types/instagram/download.d.mts

@@ -0,0 +1 @@
+export {};

package/types/instagram/index.d.mts

@@ -0,0 +1,13 @@
+/**
+ * Downloads the photos of Instagram posts (and optionally a whole profile) through toolzu.com.
+ * At least one of `links`, `file` or `profile` is required. Resolves to a summary whose `failed`
+ * lists `{ item, reason }` and whose `found` is present only in a dry run. Throws a TablefactsError
+ * on invalid arguments (code 'EUSAGE', with `option`), a missing playwright ('EDEPENDENCY') or a
+ * browser that cannot be reached.
+ * `out`, `file` and `userDataDir` resolve against `projectDir` (default: TABLEFACTS_PROJECT or the
+ * current folder). `viaGoogle` reaches toolzu through a Google search. `log(message, level)` gets
+ * level 'info', 'warn' or 'error'.
+ * @param {import('../lib/types.mjs').InstagramOptions} options
+ * @returns {Promise<import('../lib/types.mjs').PhotoSummary>}
+ */
+export declare function downloadInstagram({ links, file, out, profile, pages, cdp, edgeDir, viaGoogle, browser, userDataDir, projectDir, headed, dryRun, debug, log: logOption, }?: import('../lib/types.mjs').InstagramOptions): Promise<import('../lib/types.mjs').PhotoSummary>;

package/types/instagram/links.d.mts

@@ -0,0 +1,14 @@
+export declare function parseArgs(argv: any): {
+    links: never[];
+    headed: boolean;
+    dryRun: boolean;
+    debug: boolean;
+};
+export declare function normalize(raw: any): {
+    shortcode: string;
+    url: string;
+} | null;
+export declare function parseProfile(raw: any): {
+    user: any;
+    url: string;
+} | null;

package/types/instagram/record.d.mts

@@ -0,0 +1 @@
+export {};

package/types/lib/edge.d.mts

@@ -0,0 +1,14 @@
+export declare const DEFAULT_CDP = "http://localhost:9222";
+export declare const DEFAULT_EDGE_DIR = "C:\\ig-edge";
+/**
+ * Whether a browser answers on the DevTools address.
+ * @param {string} cdp e.g. http://localhost:9222
+ * @returns {Promise<boolean>}
+ */
+export declare function cdpUp(cdp: string): Promise<boolean>;
+/**
+ * @param {string} cdp DevTools address; only localhost can be started
+ * @param {string} [edgeDir] Edge profile folder. Default C:\ig-edge
+ * @returns {Promise<void>}
+ */
+export declare function ensureEdge(cdp: string, edgeDir?: string): Promise<void>;

package/types/lib/env.d.mts

@@ -0,0 +1,12 @@
+/** `env` when given, else process.env. */
+export declare const resolveEnv: (env: any) => any;
+/** Loads `files` (default: the project's .env files) into `env`. The first file that sets a variable wins. Returns the files it read. */
+export declare function loadEnvFiles(files?: string[], env?: NodeJS.ProcessEnv): string[];
+/**
+ * Loads the project's env files (`projectDir`, else TABLEFACTS_PROJECT or the current folder) or
+ * exactly `files` into `env` (default process.env). Never overrides a variable that is already set.
+ * Returns the files it read.
+ * @param {import('./types.mjs').LoadEnvOptions} [options]
+ * @returns {string[]}
+ */
+export declare function loadEnv({ projectDir, files, env }?: import('./types.mjs').LoadEnvOptions): string[];

package/types/lib/errors.d.mts

@@ -0,0 +1,25 @@
+export declare class TablefactsError extends Error {
+    /** @type {import('./types.mjs').TablefactsErrorCode} */
+    code: import('./types.mjs').TablefactsErrorCode;
+    /** The library option the error is about, when it is about one. @type {string=} */
+    option: string | undefined;
+    /**
+     * @param {string} message
+     * @param {import('./types.mjs').TablefactsErrorCode} code
+     * @param {{ cause?: unknown, option?: string }} [options] `option` names the library option the message is about.
+     */
+    constructor(message: string, code: import('./types.mjs').TablefactsErrorCode, options?: {
+        cause?: unknown;
+        option?: string;
+    });
+}
+/** Bad or missing arguments: the CLIs exit with 2. */
+export declare const usageError: (message: any) => TablefactsError;
+/** The exit code a CLI uses for an error it reports: 2 for bad arguments (EUSAGE), 1 for anything else. */
+export declare const exitCodeFor: (err: any) => 1 | 2;
+/** "fooBar" -> "foo-bar". */
+export declare const kebab: (name: any) => string;
+/** An error about a library option. Write the option in backticks in the message, e.g. "`out` is required". */
+export declare const optionError: (option: any, message: any, code?: string) => TablefactsError;
+/** The message with `option` tokens replaced by CLI flag text from `flags` ({ out: '--out <folder>' }). */
+export declare const cliMessage: (err: any, flags?: {}) => string;

package/types/lib/files.d.mts

@@ -0,0 +1 @@
+export declare function exists(path: any): Promise<boolean>;

package/types/lib/images.d.mts

@@ -0,0 +1,6 @@
+/** Throws EFAILED unless the response is a good image: ok, an image/* content type and at least `minBytes` bytes. */
+export declare function assertImageResponse({ ok, status, contentType, bytes, minBytes, label }?: {
+    minBytes?: number | undefined;
+}): void;
+/** Writes the bytes to `file`, creating its folder first. */
+export declare function writeImage(file: any, bytes: any): Promise<void>;

package/types/lib/log.d.mts

@@ -0,0 +1,5 @@
+export declare const noopLog: () => void;
+/** For CLIs: info goes to stdout, warn and error to stderr. */
+export declare const consoleLog: (message: any, level?: string) => void;
+/** Always returns a function that is called as log(message, level); tolerates a missing log or one that takes one argument. */
+export declare function normalizeLog(log: any): (message: any, level?: string) => void;

package/types/lib/photos.d.mts

@@ -0,0 +1,30 @@
+export declare const DELAY_MS = 2500;
+/** { saved, skipped, failed: [{ item, reason }], found? } where `found` ([{ name, url }]) exists only in a dry run. */
+export declare function newSummary({ dryRun }?: {
+    dryRun?: boolean | undefined;
+}): {
+    saved: number;
+    skipped: number;
+    failed: never[];
+};
+/** Resolves the output folder and creates it (not in a dry run), plus <out>/_debug when `debug`. */
+export declare function prepareOut({ out, dryRun, debug, projectDir }?: {
+    debug?: boolean | undefined;
+    dryRun?: boolean | undefined;
+}): Promise<{
+    outDir: string;
+    debugDir: string | null;
+}>;
+/**
+ * Dry run: records { name, url } in summary.found. File already there: counts it as skipped.
+ * Otherwise awaits `save({ file, name, url })` and counts it as saved. A save error is not caught.
+ */
+export declare function storeFile({ outDir, name, url, dryRun, summary, log, save }: {
+    dryRun: any;
+    log: any;
+    name: any;
+    outDir: any;
+    save: any;
+    summary: any;
+    url: any;
+}): Promise<void>;