soustack 0.1.1 → 0.1.2

package/dist/index.mjs

@@ -1290,6 +1290,8 @@ function extractRecipeNode(input) {
 function hasRecipeType(value) {
   if (!value) return false;
   const types = Array.isArray(value) ? value : [value];
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "fromSchemaOrg.ts:95", message: "hasRecipeType check", data: { types, typesLower: types.map((t) => typeof t === "string" ? t.toLowerCase() : t), isMatch: types.some((e) => typeof e === "string" && e.toLowerCase() === "recipe") }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A" }) }).catch(() => {
+  });
   return types.some(
     (entry) => typeof entry === "string" && entry.toLowerCase() === "recipe"
   );
@@ -1646,18 +1648,26 @@ var DEFAULT_USER_AGENTS = [
   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
   "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0"
 ];
-var fetchImpl = null;
-async function ensureFetch() {
-  if (!fetchImpl) {
-    fetchImpl = import('node-fetch').then((mod) => mod.default);
-  }
-  return fetchImpl;
-}
 function chooseUserAgent(provided) {
   if (provided) return provided;
   const index = Math.floor(Math.random() * DEFAULT_USER_AGENTS.length);
   return DEFAULT_USER_AGENTS[index];
 }
+function resolveFetch(fetchFn) {
+  if (fetchFn) {
+    return fetchFn;
+  }
+  const globalFetch = globalThis.fetch;
+  if (!globalFetch) {
+    throw new Error(
+      "A global fetch implementation is not available. Provide window.fetch in browsers or upgrade to Node 18+."
+    );
+  }
+  return globalFetch;
+}
+function isBrowserEnvironment() {
+  return typeof globalThis.document !== "undefined";
+}
 function isClientError(error) {
   if (typeof error.status === "number") {
     return error.status >= 400 && error.status < 500;
@@ -1671,25 +1681,40 @@ async function fetchPage(url, options = {}) {
   const {
     timeout = 1e4,
     userAgent,
-    maxRetries = 2
+    maxRetries = 2,
+    fetchFn
   } = options;
   let lastError = null;
+  const resolvedFetch = resolveFetch(fetchFn);
+  const isBrowser2 = isBrowserEnvironment();
   for (let attempt = 0; attempt <= maxRetries; attempt++) {
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeout);
     try {
-      const fetch = await ensureFetch();
       const headers = {
-        "User-Agent": chooseUserAgent(userAgent),
         Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
         "Accept-Language": "en-US,en;q=0.5"
       };
-      const response = await fetch(url, {
+      if (!isBrowser2) {
+        headers["User-Agent"] = chooseUserAgent(userAgent);
+      }
+      const requestInit = {
         headers,
         signal: controller.signal,
         redirect: "follow"
-      });
+      };
+      const response = await resolvedFetch(url, requestInit);
       clearTimeout(timeoutId);
+      if (response && (typeof process === "undefined" || process.env.NODE_ENV !== "test")) {
+        try {
+          const globalFetch = typeof globalThis !== "undefined" && typeof globalThis.fetch !== "undefined" ? globalThis.fetch : null;
+          if (globalFetch) {
+            globalFetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/fetch.ts:63", message: "fetch response", data: { url, status: response.status, statusText: response.statusText, ok: response.ok, isNYTimes: url.includes("nytimes.com") }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "B" }) }).catch(() => {
+            });
+          }
+        } catch {
+        }
+      }
       if (!response.ok) {
         const error = new Error(
           `HTTP ${response.status}: ${response.statusText}`
@@ -1697,7 +1722,18 @@ async function fetchPage(url, options = {}) {
         error.status = response.status;
         throw error;
       }
-      return await response.text();
+      const html = await response.text();
+      if (typeof process === "undefined" || process.env.NODE_ENV !== "test") {
+        try {
+          const globalFetch = typeof globalThis !== "undefined" && typeof globalThis.fetch !== "undefined" ? globalThis.fetch : null;
+          if (globalFetch) {
+            globalFetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/fetch.ts:75", message: "HTML received", data: { htmlLength: html.length, hasLoginPage: html.toLowerCase().includes("login") || html.toLowerCase().includes("sign in"), hasRecipeData: html.includes("application/ld+json") || html.includes("schema.org/Recipe") }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "B,D" }) }).catch(() => {
+            });
+          }
+        } catch {
+        }
+      }
+      return html;
     } catch (err) {
       clearTimeout(timeoutId);
       lastError = err instanceof Error ? err : new Error(String(err));
@@ -1724,6 +1760,8 @@ function isRecipeNode(value) {
     return false;
   }
   const type = value["@type"];
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/utils.ts:14", message: "isRecipeNode check", data: { type, typeLower: typeof type === "string" ? type.toLowerCase() : Array.isArray(type) ? type.map((t) => typeof t === "string" ? t.toLowerCase() : t) : void 0, isMatch: typeof type === "string" ? RECIPE_TYPES.has(type.toLowerCase()) : Array.isArray(type) ? type.some((e) => typeof e === "string" && RECIPE_TYPES.has(e.toLowerCase())) : false }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A" }) }).catch(() => {
+  });
   if (typeof type === "string") {
     return RECIPE_TYPES.has(type.toLowerCase());
   }
@@ -1751,14 +1789,20 @@ function normalizeText(value) {
 function extractJsonLd(html) {
   const $ = load(html);
   const scripts = $('script[type="application/ld+json"]');
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/jsonld.ts:8", message: "JSON-LD scripts found", data: { scriptCount: scripts.length }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "C,D" }) }).catch(() => {
+  });
   const candidates = [];
   scripts.each((_, element) => {
     const content = $(element).html();
     if (!content) return;
     const parsed = safeJsonParse(content);
     if (!parsed) return;
+    fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/jsonld.ts:18", message: "JSON-LD parsed", data: { hasGraph: !!(parsed && typeof parsed === "object" && "@graph" in parsed), type: parsed && typeof parsed === "object" && "@type" in parsed ? parsed["@type"] : void 0 }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A,C" }) }).catch(() => {
+    });
     collectCandidates(parsed, candidates);
   });
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/jsonld.ts:22", message: "JSON-LD candidates", data: { candidateCount: candidates.length, candidateTypes: candidates.map((c) => c["@type"]) }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A,C" }) }).catch(() => {
+  });
   return candidates[0] ?? null;
 }
 function collectCandidates(payload, bucket) {
@@ -1830,13 +1874,123 @@ function findPropertyValue($, context, prop) {
   return normalizeText(node.attr("content")) || normalizeText(node.attr("href")) || normalizeText(node.attr("src")) || normalizeText(node.text());
 }
 
+// src/scraper/extractors/browser.ts
+var SIMPLE_PROPS2 = ["name", "description", "image", "recipeYield", "prepTime", "cookTime", "totalTime"];
+function extractRecipeBrowser(html) {
+  const jsonLdRecipe = extractJsonLdBrowser(html);
+  if (jsonLdRecipe) {
+    return { recipe: jsonLdRecipe, source: "jsonld" };
+  }
+  const microdataRecipe = extractMicrodataBrowser(html);
+  if (microdataRecipe) {
+    return { recipe: microdataRecipe, source: "microdata" };
+  }
+  return { recipe: null, source: null };
+}
+function extractJsonLdBrowser(html) {
+  if (typeof globalThis.DOMParser === "undefined") {
+    return null;
+  }
+  const parser = new globalThis.DOMParser();
+  const doc = parser.parseFromString(html, "text/html");
+  const scripts = doc.querySelectorAll('script[type="application/ld+json"]');
+  const candidates = [];
+  scripts.forEach((script) => {
+    const content = script.textContent;
+    if (!content) return;
+    const parsed = safeJsonParse(content);
+    if (!parsed) return;
+    collectCandidates2(parsed, candidates);
+  });
+  return candidates[0] ?? null;
+}
+function extractMicrodataBrowser(html) {
+  if (typeof globalThis.DOMParser === "undefined") {
+    return null;
+  }
+  const parser = new globalThis.DOMParser();
+  const doc = parser.parseFromString(html, "text/html");
+  const recipeEl = doc.querySelector('[itemscope][itemtype*="schema.org/Recipe"]');
+  if (!recipeEl) {
+    return null;
+  }
+  const recipe = {
+    "@type": "Recipe"
+  };
+  SIMPLE_PROPS2.forEach((prop) => {
+    const value = findPropertyValue2(recipeEl, prop);
+    if (value) {
+      recipe[prop] = value;
+    }
+  });
+  const ingredients = [];
+  recipeEl.querySelectorAll('[itemprop="recipeIngredient"]').forEach((el) => {
+    const text = normalizeText(
+      el.getAttribute("content") || el.textContent || void 0
+    );
+    if (text) ingredients.push(text);
+  });
+  if (ingredients.length) {
+    recipe.recipeIngredient = ingredients;
+  }
+  const instructions = [];
+  recipeEl.querySelectorAll('[itemprop="recipeInstructions"]').forEach((el) => {
+    const text = normalizeText(el.getAttribute("content")) || normalizeText(el.querySelector('[itemprop="text"]')?.textContent || void 0) || normalizeText(el.textContent || void 0);
+    if (text) instructions.push(text);
+  });
+  if (instructions.length) {
+    recipe.recipeInstructions = instructions;
+  }
+  if (recipe.name || ingredients.length) {
+    return recipe;
+  }
+  return null;
+}
+function findPropertyValue2(context, prop) {
+  const node = context.querySelector(`[itemprop="${prop}"]`);
+  if (!node) return void 0;
+  return normalizeText(node.getAttribute("content")) || normalizeText(node.getAttribute("href")) || normalizeText(node.getAttribute("src")) || normalizeText(node.textContent || void 0);
+}
+function collectCandidates2(payload, bucket) {
+  if (!payload) return;
+  if (Array.isArray(payload)) {
+    payload.forEach((entry) => collectCandidates2(entry, bucket));
+    return;
+  }
+  if (typeof payload !== "object") {
+    return;
+  }
+  if (isRecipeNode(payload)) {
+    bucket.push(payload);
+    return;
+  }
+  const graph = payload["@graph"];
+  if (Array.isArray(graph)) {
+    graph.forEach((entry) => collectCandidates2(entry, bucket));
+  }
+}
+
 // src/scraper/extractors/index.ts
+function isBrowser() {
+  try {
+    return typeof globalThis.DOMParser !== "undefined";
+  } catch {
+    return false;
+  }
+}
 function extractRecipe(html) {
+  if (isBrowser()) {
+    return extractRecipeBrowser(html);
+  }
   const jsonLdRecipe = extractJsonLd(html);
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/index.ts:6", message: "JSON-LD extraction result", data: { hasJsonLd: !!jsonLdRecipe }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "C,D" }) }).catch(() => {
+  });
   if (jsonLdRecipe) {
     return { recipe: jsonLdRecipe, source: "jsonld" };
   }
   const microdataRecipe = extractMicrodata(html);
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/index.ts:12", message: "Microdata extraction result", data: { hasMicrodata: !!microdataRecipe }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "D" }) }).catch(() => {
+  });
   if (microdataRecipe) {
     return { recipe: microdataRecipe, source: "microdata" };
   }
@@ -1845,12 +1999,31 @@ function extractRecipe(html) {
 
 // src/scraper/index.ts
 async function scrapeRecipe(url, options = {}) {
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/index.ts:7", message: "scrapeRecipe entry", data: { url, hasOptions: !!options }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A,B,C,D,E" }) }).catch(() => {
+  });
   const html = await fetchPage(url, options);
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/index.ts:9", message: "HTML fetched", data: { htmlLength: html?.length, htmlPreview: html?.substring(0, 200) }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "B" }) }).catch(() => {
+  });
   const { recipe } = extractRecipe(html);
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/index.ts:11", message: "extractRecipe result", data: { hasRecipe: !!recipe, recipeType: recipe?.["@type"], recipeName: recipe?.name }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A,C,D" }) }).catch(() => {
+  });
   if (!recipe) {
     throw new Error("No Schema.org recipe data found in page");
   }
   const soustackRecipe = fromSchemaOrg(recipe);
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/index.ts:17", message: "fromSchemaOrg result", data: { hasSoustackRecipe: !!soustackRecipe, soustackRecipeName: soustackRecipe?.name }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A" }) }).catch(() => {
+  });
+  if (!soustackRecipe) {
+    throw new Error("Schema.org data did not include a valid recipe");
+  }
+  return soustackRecipe;
+}
+function extractRecipeFromHTML(html) {
+  const { recipe } = extractRecipe(html);
+  if (!recipe) {
+    throw new Error("No Schema.org recipe data found in HTML");
+  }
+  const soustackRecipe = fromSchemaOrg(recipe);
   if (!soustackRecipe) {
     throw new Error("Schema.org data did not include a valid recipe");
   }
@@ -2099,6 +2272,6 @@ function wordToNumber(word) {
   return null;
 }
 
-export { formatDuration, formatYield2 as formatYield, fromSchemaOrg, normalizeIngredientInput, normalizeYield, parseDuration, parseHumanDuration, parseIngredient, parseIngredientLine, parseIngredients, parseYield2 as parseYield, scaleRecipe, scrapeRecipe, smartParseDuration, toSchemaOrg, validateRecipe };
+export { extractRecipeFromHTML, formatDuration, formatYield2 as formatYield, fromSchemaOrg, normalizeIngredientInput, normalizeYield, parseDuration, parseHumanDuration, parseIngredient, parseIngredientLine, parseIngredients, parseYield2 as parseYield, scaleRecipe, scrapeRecipe, smartParseDuration, toSchemaOrg, validateRecipe };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map