tablefacts 0.1.0 → 0.3.0

package/src/research/lib/report.mjs

@@ -33,17 +33,54 @@ const ROWS = [
 
 const cell = (s) => String(s).replace(/\|/g, "\\|");
 
-export function renderReport(p, { notes, photos }) {
+// Web text lands in a file agents are told to act on: collapse newlines and
+// neutralise backticks so third-party text cannot forge headings or a code fence.
+const flat = (s) => String(s ?? "").replace(/\s+/g, " ").replace(/`/g, "'").trim();
+
+/**
+ * Splits the fields into what was discovered from a source, what was only echoed
+ * back from the user's own input, and low-confidence guesses, so a summary can
+ * say "N facts, M from your input" instead of counting all of them as found.
+ */
+export function fieldSummary(p) {
+  const held = Object.entries(p.fields ?? {}).filter(([, f]) => {
+    const v = f?.value;
+    return v !== undefined && v !== null && v !== "" && !(Array.isArray(v) && !v.length);
+  });
+  const parts = (f) => String(f.source ?? "").split(" + ").map((s) => s.trim()).filter(Boolean);
+  return {
+    discovered: held.filter(([, f]) => f.confidence !== "low" && parts(f).some((s) => s !== "you")).map(([k]) => k),
+    echoed: held.filter(([, f]) => {
+      const sources = parts(f);
+      return f.confidence !== "low" && sources.length > 0 && sources.every((s) => s === "you");
+    }).map(([k]) => k),
+    guessed: held.filter(([, f]) => f.confidence === "low").map(([k]) => k),
+  };
+}
+
+/** The CLI summary: what came from a source, what was only echoed, what was a low guess. */
+export function summaryLines(p) {
+  const { discovered, echoed, guessed } = fieldSummary(p);
+  const list = (a) => (a.length ? a.join(", ") : "none");
+  const lines = [`Found ${discovered.length} fact(s) from sources: ${list(discovered)}`];
+  if (echoed.length) lines.push(`Echoed ${echoed.length} from your input: ${list(echoed)}`);
+  if (guessed.length) lines.push(`Not counted (low-confidence guess): ${list(guessed)}`);
+  return lines;
+}
+
+export function renderReport(p, { notes, photos, kept = [] }) {
   const out = [];
   const q = p.query;
-  out.push(`# Research: ${q.name}${q.location ? `, ${q.location}` : ""}`, "", `Generated ${p.generatedAt}. Everything here is **unconfirmed**: check it with the client and copy the answers into \`BRIEF.md\`. Sources disagree often, and the confidence column says how many agreed.`, "");
+  out.push(`# Research: ${flat(q.name)}${q.location ? `, ${flat(q.location)}` : ""}`, "", `Generated ${p.generatedAt}. Everything here is **unconfirmed**: check it with the client and copy the answers into \`BRIEF.md\`. Sources disagree often, and the confidence column says how many agreed.`, "");
+  for (const k of kept) out.push(`> **Kept for a manual read:** ${k.label} (${k.url}) — could not be read (${k.reason}). Nothing was extracted from it; open it yourself.`, "");
   if (p.warnings.length) out.push("## Warnings", "", ...p.warnings.map((w) => `- ${w}`), "");
   if (notes.length) out.push("## What each source did", "", ...notes.map((n) => `- ${n}`), "");
 
   out.push("## Facts", "", "| Item | Value | Source | Confidence | Goes in |", "| --- | --- | --- | --- | --- |");
   for (const [label, key, goes] of ROWS) {
     const f = p.fields[key];
-    out.push(`| ${label} | ${f ? cell(val(f)) : "_not found_"} | ${f?.source ?? ""} | ${f?.confidence ?? ""} | ${goes} |`);
+    const confidence = f ? (f.confidence === "low" ? "low (guess)" : f.confidence) : "";
+    out.push(`| ${label} | ${f ? cell(val(f)) : "_not found_"} | ${f?.source ?? ""} | ${confidence} | ${goes} |`);
   }
   out.push("");
 
@@ -75,7 +112,12 @@ export function renderReport(p, { notes, photos }) {
   out.push("");
 
   if (p.ratings.length) out.push("## Ratings (context only)", "", ...p.ratings.map((r) => `- ${r.source}: ${r.value}${r.count ? ` (${r.count} reviews)` : ""}`), "");
-  if (p.social.instagram && !p.social.instagram.blocked) out.push(`Instagram @${p.social.instagram.handle}: ${p.social.instagram.followers} followers, ${p.social.instagram.posts} posts.`, "");
+  const ig = p.social.instagram;
+  if (ig && !ig.blocked) {
+    const counts = ig.followers || ig.posts ? `${ig.followers || "?"} followers, ${ig.posts || "?"} posts.` : "";
+    const note = ig.partial ? "Bio from a search snippet (the profile itself was walled)." : counts ? "" : "Profile card read; followers and bio are not public.";
+    out.push(`Instagram @${ig.handle}: ${[counts, note].filter(Boolean).join(" ")}`.trim(), "");
+  }
 
   out.push("## Images", "");
   out.push(`- Website images found: ${p.images.site.length}; link-in-bio: ${p.images.hub.length}; Google photos available: ${p.images.googlePhotos}.`);
@@ -86,13 +128,21 @@ export function renderReport(p, { notes, photos }) {
 
   out.push("## Material for the copy", "", "Facts and tone only. Rewrite it in the template's voice in both languages; do not paste it.", "");
   const c = p.copySources;
-  for (const [label, text] of [["Google summary", c.googleSummary], ["Site description", c.siteDescription], ["Instagram bio", c.instagramBio], ["TripAdvisor", c.tripadvisor]]) if (text) out.push(`- **${label}:** ${text}`);
-  if (c.headings.length) out.push(`- **Site headings:** ${c.headings.slice(0, 12).join(" / ")}`);
-  for (const t of c.paragraphs.slice(0, 6)) out.push(`  - ${t}`);
+  for (const [label, text] of [["Google summary", c.googleSummary], ["Site description", c.siteDescription], ["Instagram bio", c.instagramBio], ["TripAdvisor", c.tripadvisor]]) if (text) out.push(`- **${label}:** ${flat(text)}`);
+  if (c.headings.length) out.push(`- **Site headings:** ${c.headings.slice(0, 12).map(flat).join(" / ")}`);
+  for (const t of c.paragraphs.slice(0, 6)) out.push(`  - ${flat(t)}`);
   out.push("");
 
   const missing = ROWS.filter(([, key]) => !p.fields[key] && ["name", "street", "coordinates", "whatsapp", "instagram", "reserveUrl", "cuisines"].includes(key)).map(([l]) => l);
-  out.push("## Ask the client", "", ...[...missing, "Opening hours incl. holidays", "Logo as SVG and original photos", "The story, signature dishes and events", "Production domain"].filter((v, i, a) => a.indexOf(v) === i && (v !== "Opening hours incl. holidays" || true)).map((m) => `- ${m}`), "");
+  const ask = !p.fields.website && !p.fields.mapsUrl
+    ? "Send the restaurant's website or a Google Maps link — that unlocks the address, map point and place ID."
+    : !p.fields.coordinates
+      ? "A Google Maps link would pin the exact location."
+      : !p.fields.whatsapp
+        ? "Ask for the WhatsApp number taken with reservations, and confirm it accepts messages."
+        : "Ask for the current website and social profiles to cross-check the rest.";
+  const asks = ["Opening hours incl. holidays", "Logo as SVG and original photos", "The story, signature dishes and events", "Production domain"];
+  out.push("## Ask the client", "", `- **Best next question:** ${ask}`, ...[...new Set([...missing, ...asks])].map((m) => `- ${m}`), "");
   out.push("## Next", "", "```bash", `npm run setup < .tablefacts/research/${q.slug}/setup-answers.txt`, "git diff   # review before keeping it", "```", "", "Blank lines in that file keep the template's current value, so anything not found stays a placeholder.", "");
   return out.join("\n");
 }
@@ -100,19 +150,22 @@ export function renderReport(p, { notes, photos }) {
 /** One line per prompt of `data/scripts/setup.mjs`, in its order. Blank keeps the current value. */
 export function setupAnswers(p) {
   const f = p.fields;
-  const ig = f.instagram?.value;
+  // A low-confidence guess stays blank so setup keeps the template's placeholder,
+  // instead of a guess being written in as though it were a discovered fact.
+  const sure = (field) => (field && field.confidence !== "low" ? field : null);
+  const ig = sure(f.instagram)?.value;
   return [
-    val(f.name),
+    val(sure(f.name)),
     "", // production URL: the new domain, not the current website
-    f.reserveUrl?.value ?? "",
-    f.whatsapp?.display ?? "",
+    sure(f.reserveUrl)?.value ?? "",
+    sure(f.whatsapp)?.display ?? "",
     ig ? `@${ig}` : "",
-    val(f.street),
-    val(f.locality),
-    val(f.region),
-    val(f.country),
-    f.coordinates?.value?.lat ?? "",
-    f.coordinates?.value?.lng ?? "",
-    val(f.cuisines),
+    val(sure(f.street)),
+    val(sure(f.locality)),
+    val(sure(f.region)),
+    val(sure(f.country)),
+    sure(f.coordinates)?.value?.lat ?? "",
+    sure(f.coordinates)?.value?.lng ?? "",
+    val(sure(f.cuisines)),
   ].join("\n") + "\n";
 }

package/src/research/lib/search.mjs

@@ -0,0 +1,127 @@
+// Key-free web search, used when Google Places and OpenStreetMap find no place
+// (or no website): a bare name + place otherwise yields an empty report. It only
+// surfaces candidate links (website, Instagram, TripAdvisor, Maps, hub); the rest
+// of the pipeline follows them. DuckDuckGo's Lite endpoint is tried first (the
+// same results in simpler HTML, no JavaScript), then the html endpoint. No key
+// and no login. A bot wall makes the source fail like any other.
+import { fetchText, fold, sameText, tokens, decodeHtml, instagramHandle } from "./util.mjs";
+
+const DDG = [
+  "https://lite.duckduckgo.com/lite/",
+  "https://html.duckduckgo.com/html/",
+];
+
+// Hosts that are directories or social pages, never the restaurant's own site.
+const NOT_WEBSITE =
+  /(duckduckgo|bing|google|youtube|facebook|instagram|tiktok|twitter|(^|\.)x\.com$|tripadvisor|yelp|wikipedia|foursquare|mapquest|waze|trip\.com|expedia|booking|airbnb|justdial|zomato|rappi|ubereats|pedidosya|doordash|grubhub|glovo|opentable|thefork|eltenedor|exploretock|sevenrooms|covermanager|quandoo|linktr|beacons|linkin\.bio|lnk\.bio|taplink|wa\.me|whatsapp)/i;
+const INSTAGRAM = /(^|\.)instagram\.com$/i;
+const TRIPADVISOR = /tripadvisor\./i;
+const FACEBOOK = /(^|\.)facebook\.com$|(^|\.)fb\.com$/i;
+const TIKTOK = /(^|\.)tiktok\.com$/i;
+const MAPS = /maps\.app\.goo\.gl|(^|\.)google\.[a-z.]+\/maps|maps\.google\.com|goo\.gl\/maps/i;
+const HUBS = /(linktr\.ee|beacons\.ai|bio\.link|lnk\.bio|linktree\.com|taplink|campsite\.bio|solo\.to|linkin\.bio|flow\.page)/i;
+
+const textOf = (s) => decodeHtml(String(s).replace(/<[^>]+>/g, " ")).replace(/\s+/g, " ").trim();
+
+/** Unwraps DuckDuckGo's `//duckduckgo.com/l/?uddg=<url>` redirect to the real result URL. */
+export function resultUrl(href) {
+  if (!href) return null;
+  let u;
+  try { u = new URL(href, "https://html.duckduckgo.com"); } catch { return null; }
+  if (/(^|\.)duckduckgo\.com$/i.test(u.hostname)) {
+    const target = u.searchParams.get("uddg");
+    if (!target) return null;
+    try { return new URL(target).href; } catch { return null; }
+  }
+  return u.protocol === "http:" || u.protocol === "https:" ? u.href : null;
+}
+
+// One pass over the page, matching a result link or a snippet cell in order, so a
+// result with no snippet cannot borrow the next result's text.
+const RESULT_TOKEN = /<a\b([^>]*\b(?:result__a|result-link)\b[^>]*)>([\s\S]*?)<\/a>|<([a-z]+)\b[^>]*\b(?:result__snippet|result-snippet)\b[^>]*>([\s\S]*?)<\/\3>/gi;
+
+/** Pulls { url, title, snippet } out of DuckDuckGo's HTML result page (no DOM parser). */
+export function parseSearchResults(html) {
+  const out = [];
+  const seen = new Set();
+  let current = -1;
+  for (const m of html.matchAll(RESULT_TOKEN)) {
+    if (m[1] !== undefined) {
+      const href = m[1].match(/href=["']([^"']+)["']/i)?.[1] ?? "";
+      const url = resultUrl(href);
+      if (!url || seen.has(url)) { current = -1; continue; }
+      seen.add(url);
+      out.push({ url, title: textOf(m[2]), snippet: "" });
+      current = out.length - 1;
+    } else if (current >= 0 && !out[current].snippet) {
+      out[current].snippet = textOf(m[4]);
+    }
+  }
+  return out;
+}
+
+const hostOf = (url) => { try { return new URL(url).hostname.replace(/^www\./, ""); } catch { return ""; } };
+
+/**
+ * Sorts results into the links the pipeline can follow. `website` is the best
+ * non-directory result whose title/host looks like the restaurant's name; a
+ * result that only matches the place is kept as a weaker candidate.
+ */
+export function classifyResults(results, { name = "" } = {}) {
+  const links = { website: [], instagram: [], tripadvisor: [], facebook: [], tiktok: [], maps: [], hubs: [] };
+  const add = (k, v) => { if (v && !links[k].includes(v)) links[k].push(v); };
+  const candidates = [];
+  for (const { url, title, snippet } of results) {
+    const host = hostOf(url);
+    if (INSTAGRAM.test(host)) {
+      const handle = instagramHandle(url);
+      if (handle) add("instagram", handle);
+    } else if (TRIPADVISOR.test(host)) add("tripadvisor", url);
+    else if (FACEBOOK.test(host)) add("facebook", url);
+    else if (TIKTOK.test(host)) add("tiktok", url);
+    else if (MAPS.test(url)) add("maps", url);
+    else if (HUBS.test(host)) add("hubs", url);
+    else if (NOT_WEBSITE.test(host)) continue;
+    else {
+      const strong = sameText(name, title) || tokens(name).some((t) => fold(host).includes(t));
+      const score = tokens(name).filter((t) => fold(`${title} ${host} ${snippet}`).includes(t)).length;
+      candidates.push({ url, title, snippet, score, strong });
+    }
+  }
+  candidates.sort((a, b) => Number(b.strong) - Number(a.strong) || b.score - a.score);
+  for (const c of candidates) add("website", c.url);
+  return {
+    website: candidates.find((c) => c.strong)?.url ?? candidates[0]?.url ?? "",
+    instagram: links.instagram[0] ?? "",
+    tripadvisor: links.tripadvisor[0] ?? "",
+    facebook: links.facebook[0] ?? "",
+    tiktok: links.tiktok[0] ?? "",
+    mapsUrl: links.maps[0] ?? "",
+    hubs: links.hubs,
+    candidates: candidates.slice(0, 5).map(({ url, title }) => ({ url, title })),
+  };
+}
+
+/**
+ * Runs the search for a restaurant and returns its candidate links. Tries every
+ * endpoint until one returns results; if all answer but none has results, the
+ * empty result is returned so the caller still says "no results" rather than
+ * "failed". Throws only when no endpoint answers.
+ */
+export async function searchWeb({ name, location = "" }) {
+  const query = [name, location, "restaurant"].filter(Boolean).join(" ");
+  const qs = new URLSearchParams({ q: query }).toString();
+  let lastError, empty;
+  for (const base of DDG) {
+    let r;
+    try { r = await fetchText(`${base}?${qs}`, { headers: { accept: "text/html" } }); }
+    catch (e) { lastError = e; continue; }
+    if (!r.ok) { lastError = new Error(`DuckDuckGo ${r.status}`); continue; }
+    const results = parseSearchResults(r.text);
+    const found = { source: "search", url: r.url, query, results, ...classifyResults(results, { name }) };
+    if (results.length) return found;
+    empty = found;
+  }
+  if (empty) return empty;
+  throw lastError ?? new Error("DuckDuckGo search failed");
+}

package/src/research/lib/social.mjs

@@ -1,44 +1,155 @@
 // Instagram, TripAdvisor and link-in-bio hubs. All three are best-effort: they
 // sit behind login walls or bot protection, so each one returns what it could
 // read plus a `blocked` reason, and the report says what is missing.
-import { fetchText } from "./util.mjs";
+import { fetchText, sleep, decodeHtml as decodeEntities } from "./util.mjs";
 import { readPage } from "./website.mjs";
 
-/**
- * Public Instagram profile metadata. Without a login the page only exposes the
- * share card: "1,234 Followers, 56 Following, 789 Posts - Name (@handle) on
- * Instagram: "bio"". The bio often carries the phone, the area and a hub link.
- */
-export async function readInstagram(handle) {
-  const url = `https://www.instagram.com/${handle}/`;
-  let r;
-  try { r = await fetchText(url, { headers: { accept: "text/html" } }); } catch (e) { return { handle, url, blocked: e.message }; }
-  if (!r.ok) return { handle, url, blocked: `HTTP ${r.status}` };
-  const meta = (key) => r.text.match(new RegExp(`<meta[^>]+(?:property|name)=["']${key}["'][^>]+content=["']([^"']*)["']`, "i"))?.[1] ?? "";
-  const desc = meta("og:description").replace(/&quot;/g, '"').replace(/&#039;|&#x27;/g, "'").replace(/&amp;/g, "&");
-  const title = meta("og:title");
-  if (!desc && !title) return { handle, url, blocked: "login wall (no public metadata)" };
+// Instagram's own web app sends this app id with its public profile request; no
+// login is involved, and the request identifies honestly.
+const IG_APP_ID = "936619743392459";
+// A short gap between the Instagram surfaces: one walled profile should not become a burst.
+const SURFACE_PAUSE = 300;
+
+const bioLinks = (bio) => (bio.match(/https?:\/\/[^\s)]+|\b[\w-]+\.(?:com|co|link|bio|ee)\/[\w./-]+/gi) ?? [])
+  .map((l) => (/^https?:/.test(l) ? l : `https://${l}`));
+const bioPhones = (bio) => [...bio.matchAll(/\+?\d[\d\s().-]{7,}\d/g)].map((m) => m[0].trim());
+
+/** The public share card: "1,234 Followers, 56 Following, 789 Posts - Name (@handle) on Instagram: "bio"". */
+function shareCard(html) {
+  const meta = (key) => html.match(new RegExp(`<meta[^>]+(?:property|name)=["']${key}["'][^>]+content=["']([^"']*)["']`, "i"))?.[1] ?? "";
+  const desc = decodeEntities(meta("og:description"));
+  const title = decodeEntities(meta("og:title"));
+  if (!desc && !title) return null;
+  return { title, desc, image: meta("og:image") };
+}
+
+function fromCard(handle, url, card, surface) {
+  const desc = card.desc;
   const stats = desc.match(/([\d.,KMkm]+)\s+Followers?,\s*([\d.,KMkm]+)\s+Following,\s*([\d.,KMkm]+)\s+Posts?/i);
-  const bio = desc.match(/on Instagram:\s*["“]([\s\S]*?)["”]?\s*$/i)?.[1]?.trim() ?? "";
-  const links = bio.match(/https?:\/\/[^\s)]+|\b[\w-]+\.(?:com|co|link|bio|ee)\/[\w./-]+/gi) ?? [];
+  const wrapped = desc.match(/on Instagram:\s*["“]([\s\S]*?)["”]?\s*$/i)?.[1]?.trim();
+  // A bare bio snippet has no share-card wrapper; a stats-only line has no bio.
+  const bio = wrapped ?? (/Followers?,\s*[\d.,KMkm]+\s+Following/i.test(desc) ? "" : desc.trim());
   return {
-    handle, url,
-    displayName: title.replace(/\s*\(@.*$/, "").trim(),
+    handle, url, surface,
+    displayName: card.title.replace(/\s*\(@.*$/, "").trim(),
     followers: stats?.[1] ?? "",
     posts: stats?.[3] ?? "",
     bio,
-    phones: [...bio.matchAll(/\+?\d[\d\s().-]{7,}\d/g)].map((m) => m[0].trim()),
-    links: links.map((l) => (/^https?:/.test(l) ? l : `https://${l}`)),
-    image: meta("og:image"),
+    phones: bioPhones(bio),
+    links: bioLinks(bio),
+    image: card.image,
+  };
+}
+
+/** Fills in a missing bio or counts from the search-engine snippet, keeping whatever the surface itself gave. */
+function withSnippet(result, snippet) {
+  if (!snippet) return result;
+  const merged = fromCard(result.handle, result.url, { title: result.displayName ?? "", desc: snippet, image: result.image ?? "" }, result.surface);
+  return {
+    ...result,
+    followers: result.followers || merged.followers,
+    posts: result.posts || merged.posts,
+    bio: result.bio || merged.bio || snippet,
+    phones: result.phones?.length ? result.phones : merged.phones,
+    links: result.links?.length ? result.links : merged.links,
+  };
+}
+
+/** The public web_profile_info endpoint the profile page calls. No login, but often walled. */
+async function readProfileApi(handle) {
+  const url = `https://www.instagram.com/${handle}/`;
+  let r;
+  try { r = await fetchText(`https://www.instagram.com/api/v1/users/web_profile_info/?username=${encodeURIComponent(handle)}`, { headers: { accept: "application/json", "x-ig-app-id": IG_APP_ID } }); }
+  catch { return null; }
+  if (!r.ok) return null;
+  let user;
+  try { user = JSON.parse(r.text)?.data?.user; } catch { return null; }
+  if (!user) return null;
+  const bio = user.biography ?? "";
+  return {
+    handle, url, surface: "profile API",
+    displayName: user.full_name ?? "",
+    followers: user.edge_followed_by?.count != null ? String(user.edge_followed_by.count) : "",
+    posts: user.edge_owner_to_timeline_media?.count != null ? String(user.edge_owner_to_timeline_media.count) : "",
+    bio,
+    phones: bioPhones(bio),
+    links: bioLinks(bio),
+    image: user.profile_pic_url_hd ?? user.profile_pic_url ?? "",
   };
 }
 
-/** TripAdvisor: schema.org JSON-LD when the page loads; DataDome often blocks it. */
+/** oEmbed (public, though Instagram now usually wants an app token): display name only. */
+async function readOembed(handle, url) {
+  let r;
+  try { r = await fetchText(`https://api.instagram.com/oembed/?url=${encodeURIComponent(url)}`, { headers: { accept: "application/json" } }); }
+  catch { return null; }
+  if (!r.ok) return null;
+  let j;
+  try { j = JSON.parse(r.text); } catch { return null; }
+  if (!j?.author_name && !j?.title) return null;
+  return { handle, url, surface: "oembed", displayName: j.author_name ?? "", followers: "", posts: "", bio: "", phones: [], links: [], image: "" };
+}
+
+/**
+ * Public Instagram profile metadata, trying the public surfaces in turn: the
+ * share card, the web profile API, oEmbed, and finally the bio a search engine
+ * indexed (`snippet`). Only honest requests are made; a walled profile is
+ * reported as blocked, never worked around. Returns `blocked` only when none of
+ * them had anything.
+ */
+export async function readInstagram(handle, { snippet = "" } = {}) {
+  const url = `https://www.instagram.com/${handle}/`;
+  let reason = "";
+
+  try {
+    const r = await fetchText(url, { headers: { accept: "text/html" } });
+    if (!r.ok) reason = `HTTP ${r.status}`;
+    else {
+      const card = shareCard(r.text);
+      if (card) return withSnippet(fromCard(handle, url, card, "share card"), snippet);
+    }
+  } catch (e) { reason = e.message; }
+
+  await sleep(SURFACE_PAUSE);
+  const fromApi = await readProfileApi(handle);
+  if (fromApi) return withSnippet(fromApi, snippet);
+
+  await sleep(SURFACE_PAUSE);
+  const oembed = await readOembed(handle, url);
+  if (oembed) return withSnippet(oembed, snippet);
+
+  if (snippet) return { ...fromCard(handle, url, { title: "", desc: snippet, image: "" }, "search snippet"), partial: true };
+
+  return { handle, url, blocked: reason || "login wall (no public metadata on any surface)" };
+}
+
+/**
+ * Name and location out of a TripAdvisor restaurant URL slug, e.g.
+ * "...-Reviews-Maki_Bar_Medellin-Medellin_Antioquia_Department.html" gives
+ * { name: "Maki Bar Medellin", location: "Medellin Antioquia Department" }. The
+ * city/region boundary is ambiguous for multi-word cities, so the whole location
+ * is kept and the caller treats it as a hint, never as a confirmed locality.
+ */
+export function parseTripadvisorUrl(raw) {
+  let path;
+  try { path = new URL(raw).pathname; } catch { return null; }
+  const segment = path.split("/").find((s) => /-Reviews-|-ShowUserReviews-|-Management-/.test(s));
+  if (!segment) return null;
+  const after = segment.split(/-(?:Reviews|ShowUserReviews|Management)-/).pop()?.replace(/\.html?$/i, "") ?? "";
+  const [namePart = "", locationPart = ""] = after.split("-");
+  const words = (s) => decodeEntities(s).replace(/_/g, " ").replace(/\s+/g, " ").trim();
+  const name = words(namePart);
+  const location = words(locationPart);
+  return name ? { name, location } : null;
+}
+
+/** TripAdvisor: schema.org JSON-LD when the page loads; DataDome often blocks it. The URL slug is always kept. */
 export async function readTripadvisor(url, { browser } = {}) {
+  const slug = parseTripadvisorUrl(url);
   const { page, blocked } = await readPage(url, { browser });
-  if (!page) return { url, blocked };
-  if (!page.jsonld && page.textLength < 500) return { url, blocked: "bot protection (page has no data)" };
-  return { url: page.url, jsonld: page.jsonld, description: page.meta.description, hoursText: page.hoursText, images: page.images.slice(0, 10) };
+  if (!page) return { url, slug, blocked };
+  if (!page.jsonld && page.textLength < 500) return { url, slug, blocked: "bot protection (page has no data)" };
+  return { url: page.url, slug, jsonld: page.jsonld, description: page.meta.description, hoursText: page.hoursText, images: page.images.slice(0, 10) };
 }
 
 /** A link-in-bio hub (Linktree, Beacons, bio.link): its links, classified like a website's. */

package/src/research/lib/util.mjs

@@ -8,6 +8,28 @@ export const BROWSER_UA =
 
 export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
+const HTML_ENT = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " " };
+
+/** Decodes the HTML entities a page's attributes or text may hold, named and numeric. */
+export const decodeHtml = (s) => String(s).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e) => {
+  if (e[0] === "#") {
+    const n = e[1].toLowerCase() === "x" ? parseInt(e.slice(2), 16) : Number(e.slice(1));
+    return Number.isFinite(n) ? String.fromCodePoint(n) : m;
+  }
+  return HTML_ENT[e.toLowerCase()] ?? m;
+});
+
+// Instagram paths that are not a profile.
+export const IG_RESERVED = new Set(["p", "reel", "reels", "explore", "accounts", "tv", "stories", "share", "direct", "about", "legal", "web", "developer"]);
+
+/** The profile handle in an instagram.com URL, or "" for a post, reel or other non-profile path. */
+export const instagramHandle = (url) => {
+  try {
+    const h = new URL(url).pathname.split("/")[1]?.toLowerCase();
+    return h && !IG_RESERVED.has(h) && /^[a-z0-9._]+$/.test(h) ? h : "";
+  } catch { return ""; }
+};
+
 /** Runs `fn(item, index)` over `items` with at most `size` in flight; results keep the order of `items`. */
 export async function mapPool(items, size, fn) {
   const results = new Array(items.length);

package/src/research/lib/website.mjs

@@ -2,19 +2,9 @@
 // TripAdvisor page when it lets us in) with plain fetch and regexes: no HTML
 // parser dependency. Falls back to Playwright when a page renders client-side.
 import { loadPlaywright } from "../../lib/playwright.mjs";
-import { fetchText, sleep, BROWSER_UA } from "./util.mjs";
+import { fetchText, sleep, BROWSER_UA, decodeHtml as decode, instagramHandle } from "./util.mjs";
 import { fromOsm, fromSpec } from "./hours.mjs";
 
-const ENT = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " " };
-const decode = (s) =>
-  s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e) => {
-    if (e[0] === "#") {
-      const n = e[1].toLowerCase() === "x" ? parseInt(e.slice(2), 16) : Number(e.slice(1));
-      return Number.isFinite(n) ? String.fromCodePoint(n) : m;
-    }
-    return ENT[e.toLowerCase()] ?? m;
-  });
-
 const attrs = (tag) => {
   const out = {};
   for (const m of tag.matchAll(/([a-zA-Z_:][-\w:.]*)\s*(?:=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g))
@@ -25,7 +15,6 @@ const abs = (u, base) => { try { return new URL(u, base).href; } catch { return
 const strip = (s) => decode(s.replace(/<[^>]+>/g, " ")).replace(/\s+/g, " ").trim();
 const host = (u) => { try { return new URL(u).hostname.replace(/^www\./, ""); } catch { return ""; } };
 
-const IG_RESERVED = new Set(["p", "reel", "reels", "explore", "accounts", "tv", "stories", "share", "direct", "about", "legal", "web", "developer"]);
 const RESERVE = /(opentable|resy\.com|thefork|eltenedor|exploretock|sevenrooms|covermanager|quandoo|tablein|mesa247|bookatable|tablecheck|resos\.com|reservandonos|agendapro|fudo\.)/i;
 const DELIVERY = /(rappi|ubereats|pedidosya|doordash|grubhub|domicilios\.com|didi-food|glovoapp)/i;
 const HUBS = /(linktr\.ee|beacons\.ai|bio\.link|lnk\.bio|linktree\.com|taplink|campsite\.bio|solo\.to|linkin\.bio|flow\.page)/i;
@@ -102,8 +91,8 @@ export function analyzeHtml(html, base, lines = visibleLines(html)) {
       const num = (u.pathname.match(/^\/(\d{7,})/)?.[1]) ?? u.searchParams.get("phone");
       push("whatsapp", num ? num.replace(/\D/g, "") : href);
     } else if (/(^|\.)instagram\.com$/.test(h)) {
-      const handle = u.pathname.split("/")[1]?.toLowerCase();
-      if (handle && !IG_RESERVED.has(handle) && /^[a-z0-9._]+$/.test(handle)) ig.set(handle, (ig.get(handle) ?? 0) + 1);
+      const handle = instagramHandle(href);
+      if (handle) ig.set(handle, (ig.get(handle) ?? 0) + 1);
     } else if (/facebook\.com$|fb\.com$/.test(h)) push("facebook", href);
     else if (/tiktok\.com$/.test(h)) push("tiktok", href);
     else if (/tripadvisor\./.test(h)) push("tripadvisor", href);

package/src/research/research.mjs

@@ -2,6 +2,7 @@
 // Usage: tablefacts research "Restaurant name" "City, Country" [options]
 import { parseArgs } from "node:util";
 import { research } from "./index.mjs";
+import { summaryLines } from "./lib/report.mjs";
 import { loadEnv } from "../lib/env.mjs";
 import { cliMessage, exitCodeFor } from "../lib/errors.mjs";
 import { consoleLog } from "../lib/log.mjs";
@@ -10,9 +11,11 @@ const HELP = `Research a restaurant from public sources and write a profile for
 
   tablefacts research "<name>" "<city, country>" [options]
 
-Sources: Google Maps (Places API), OpenStreetMap, the restaurant's website,
-Instagram, TripAdvisor and link-in-bio pages (Linktree and similar). They find
-each other: the website or Google leads to Instagram, TripAdvisor and the hub.
+Sources: Google Maps (Places API), OpenStreetMap, a key-free web search, the
+restaurant's website, Instagram, TripAdvisor and link-in-bio pages (Linktree
+and similar). They find each other: the search, website or Google leads to
+Instagram, TripAdvisor and the hub. The web search runs when Google and
+OpenStreetMap find nothing.
 
 Options:
   --country <ISO>       country code, narrows the search (CO, MX, US...)
@@ -26,8 +29,9 @@ Options:
   --out <dir>           output folder (default .tablefacts/research/<slug>)
   -h, --help            this text
 
-Writes profile.json, report.md and setup-answers.txt. GOOGLE_PLACES_API_KEY goes
-in .env (see .env.example); without it OpenStreetMap and the web pages
+Writes profile.json, report.md and setup-answers.txt (and latest.json next to the
+folder, in the default location). GOOGLE_PLACES_API_KEY goes in .env (see
+.env.example); without it OpenStreetMap, the web search and the web pages
 still work, with fewer facts.`;
 
 const { values, positionals } = parseArgs({
@@ -52,9 +56,9 @@ try {
     tripadvisor: values.tripadvisor, linktree: values.linktree, render: values.render,
     google: !values["no-google"], photos: Number(values.photos ?? 0), out: values.out, log: consoleLog,
   });
-  const found = Object.keys(profile.fields).filter((k) => profile.fields[k]);
-  console.log(`
-Found ${found.length} fields: ${found.join(", ")}`);
+  // The lines are built in one place (summaryLines) so tests cover the exact wording the CLI prints.
+  console.log();
+  for (const line of summaryLines(profile)) console.log(line);
   for (const w of profile.warnings) console.log(`Warning: ${w}`);
   console.log(`
 Wrote ${outDir}