soustack 0.1.1 → 0.1.2

package/README.md

@@ -46,19 +46,21 @@ npm install soustack
 - **Schema.org Conversion**:
   - `fromSchemaOrg()` (Schema.org JSON-LD → Soustack)
   - `toSchemaOrg()` (Soustack → Schema.org JSON-LD)
-- **Web Scraping**: `scrapeRecipe()` fetches a recipe page and extracts Schema.org recipe data from:
-  - JSON-LD (`<script type="application/ld+json">`)
-  - Microdata (`itemscope/itemtype`)
+- **Web Scraping**:
+  - `scrapeRecipe()` fetches a recipe page and extracts Schema.org recipe data (Node.js only)
+  - `extractRecipeFromHTML()` extracts recipe data from HTML string (browser & Node.js compatible)
+  - Supports JSON-LD (`<script type="application/ld+json">`) and Microdata (`itemscope/itemtype`)
 
 ## Programmatic Usage
 
 ```ts
 import {
   scrapeRecipe,
+  extractRecipeFromHTML,
   fromSchemaOrg,
   toSchemaOrg,
   validateRecipe,
-  scaleRecipe
+  scaleRecipe,
 } from 'soustack';
 
 // Validate a Soustack recipe JSON object
@@ -67,9 +69,13 @@ validateRecipe(recipe);
 // Scale a recipe to a target yield amount (returns a "computed recipe")
 const computed = scaleRecipe(recipe, 2);
 
-// Scrape a URL into a Soustack recipe (throws if no recipe is found)
+// Scrape a URL into a Soustack recipe (Node.js only, throws if no recipe is found)
 const scraped = await scrapeRecipe('https://example.com/recipe');
 
+// Extract recipe from HTML string (browser & Node.js compatible)
+const html = await fetch('https://example.com/recipe').then((r) => r.text());
+const recipe = extractRecipeFromHTML(html);
+
 // Convert Schema.org → Soustack
 const soustack = fromSchemaOrg(schemaOrgJsonLd);
 
@@ -88,9 +94,13 @@ const soustackRecipe = fromSchemaOrg(schemaOrgJsonLd);
 const schemaOrgRecipe = toSchemaOrg(soustackRecipe);
 ```
 
-## 🧰 Scraping Options
+## 🧰 Web Scraping
+
+### Node.js: `scrapeRecipe()`
+
+`scrapeRecipe(url, options)` fetches a recipe page and extracts Schema.org data. **Node.js only** due to CORS restrictions.
 
-`scrapeRecipe(url, options)` supports basic fetch tuning:
+Options:
 
 - `timeout` (ms, default `10000`)
 - `userAgent` (string, optional)
@@ -101,10 +111,30 @@ import { scrapeRecipe } from 'soustack';
 
 const recipe = await scrapeRecipe('https://example.com/recipe', {
   timeout: 15000,
-  maxRetries: 3
+  maxRetries: 3,
 });
 ```
 
+### Browser: `extractRecipeFromHTML()`
+
+`extractRecipeFromHTML(html)` extracts recipe data from an HTML string. **Works in both browser and Node.js**. Perfect for browser usage where you fetch HTML yourself (with cookies/session for authenticated content).
+
+```ts
+import { extractRecipeFromHTML } from 'soustack';
+
+// In browser: fetch HTML yourself (bypasses CORS, uses your cookies/session)
+const response = await fetch('https://example.com/recipe');
+const html = await response.text();
+const recipe = extractRecipeFromHTML(html);
+```
+
+**Why use `extractRecipeFromHTML()` in browsers?**
+
+- ✅ No CORS issues — you fetch HTML yourself
+- ✅ Works with authenticated/paywalled content — uses browser cookies
+- ✅ Smaller bundle — no Node.js dependencies
+- ✅ Universal — works in both browser and Node.js environments
+
 ### CLI
 
 ```bash

package/dist/cli/index.js

@@ -1313,6 +1313,8 @@ function extractRecipeNode(input) {
 function hasRecipeType(value) {
   if (!value) return false;
   const types = Array.isArray(value) ? value : [value];
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "fromSchemaOrg.ts:95", message: "hasRecipeType check", data: { types, typesLower: types.map((t) => typeof t === "string" ? t.toLowerCase() : t), isMatch: types.some((e) => typeof e === "string" && e.toLowerCase() === "recipe") }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A" }) }).catch(() => {
+  });
   return types.some(
     (entry) => typeof entry === "string" && entry.toLowerCase() === "recipe"
   );
@@ -1669,18 +1671,26 @@ var DEFAULT_USER_AGENTS = [
   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
   "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0"
 ];
-var fetchImpl = null;
-async function ensureFetch() {
-  if (!fetchImpl) {
-    fetchImpl = import('node-fetch').then((mod) => mod.default);
-  }
-  return fetchImpl;
-}
 function chooseUserAgent(provided) {
   if (provided) return provided;
   const index = Math.floor(Math.random() * DEFAULT_USER_AGENTS.length);
   return DEFAULT_USER_AGENTS[index];
 }
+function resolveFetch(fetchFn) {
+  if (fetchFn) {
+    return fetchFn;
+  }
+  const globalFetch = globalThis.fetch;
+  if (!globalFetch) {
+    throw new Error(
+      "A global fetch implementation is not available. Provide window.fetch in browsers or upgrade to Node 18+."
+    );
+  }
+  return globalFetch;
+}
+function isBrowserEnvironment() {
+  return typeof globalThis.document !== "undefined";
+}
 function isClientError(error) {
   if (typeof error.status === "number") {
     return error.status >= 400 && error.status < 500;
@@ -1694,25 +1704,40 @@ async function fetchPage(url, options = {}) {
   const {
     timeout = 1e4,
     userAgent,
-    maxRetries = 2
+    maxRetries = 2,
+    fetchFn
   } = options;
   let lastError = null;
+  const resolvedFetch = resolveFetch(fetchFn);
+  const isBrowser2 = isBrowserEnvironment();
   for (let attempt = 0; attempt <= maxRetries; attempt++) {
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeout);
     try {
-      const fetch = await ensureFetch();
       const headers = {
-        "User-Agent": chooseUserAgent(userAgent),
         Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
         "Accept-Language": "en-US,en;q=0.5"
       };
-      const response = await fetch(url, {
+      if (!isBrowser2) {
+        headers["User-Agent"] = chooseUserAgent(userAgent);
+      }
+      const requestInit = {
         headers,
         signal: controller.signal,
         redirect: "follow"
-      });
+      };
+      const response = await resolvedFetch(url, requestInit);
       clearTimeout(timeoutId);
+      if (response && (typeof process === "undefined" || process.env.NODE_ENV !== "test")) {
+        try {
+          const globalFetch = typeof globalThis !== "undefined" && typeof globalThis.fetch !== "undefined" ? globalThis.fetch : null;
+          if (globalFetch) {
+            globalFetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/fetch.ts:63", message: "fetch response", data: { url, status: response.status, statusText: response.statusText, ok: response.ok, isNYTimes: url.includes("nytimes.com") }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "B" }) }).catch(() => {
+            });
+          }
+        } catch {
+        }
+      }
       if (!response.ok) {
         const error = new Error(
           `HTTP ${response.status}: ${response.statusText}`
@@ -1720,7 +1745,18 @@ async function fetchPage(url, options = {}) {
         error.status = response.status;
         throw error;
       }
-      return await response.text();
+      const html = await response.text();
+      if (typeof process === "undefined" || process.env.NODE_ENV !== "test") {
+        try {
+          const globalFetch = typeof globalThis !== "undefined" && typeof globalThis.fetch !== "undefined" ? globalThis.fetch : null;
+          if (globalFetch) {
+            globalFetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/fetch.ts:75", message: "HTML received", data: { htmlLength: html.length, hasLoginPage: html.toLowerCase().includes("login") || html.toLowerCase().includes("sign in"), hasRecipeData: html.includes("application/ld+json") || html.includes("schema.org/Recipe") }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "B,D" }) }).catch(() => {
+            });
+          }
+        } catch {
+        }
+      }
+      return html;
     } catch (err) {
       clearTimeout(timeoutId);
       lastError = err instanceof Error ? err : new Error(String(err));
@@ -1747,6 +1783,8 @@ function isRecipeNode(value) {
     return false;
   }
   const type = value["@type"];
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/utils.ts:14", message: "isRecipeNode check", data: { type, typeLower: typeof type === "string" ? type.toLowerCase() : Array.isArray(type) ? type.map((t) => typeof t === "string" ? t.toLowerCase() : t) : void 0, isMatch: typeof type === "string" ? RECIPE_TYPES.has(type.toLowerCase()) : Array.isArray(type) ? type.some((e) => typeof e === "string" && RECIPE_TYPES.has(e.toLowerCase())) : false }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A" }) }).catch(() => {
+  });
   if (typeof type === "string") {
     return RECIPE_TYPES.has(type.toLowerCase());
   }
@@ -1774,14 +1812,20 @@ function normalizeText(value) {
 function extractJsonLd(html) {
   const $ = cheerio.load(html);
   const scripts = $('script[type="application/ld+json"]');
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/jsonld.ts:8", message: "JSON-LD scripts found", data: { scriptCount: scripts.length }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "C,D" }) }).catch(() => {
+  });
   const candidates = [];
   scripts.each((_, element) => {
     const content = $(element).html();
     if (!content) return;
     const parsed = safeJsonParse(content);
     if (!parsed) return;
+    fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/jsonld.ts:18", message: "JSON-LD parsed", data: { hasGraph: !!(parsed && typeof parsed === "object" && "@graph" in parsed), type: parsed && typeof parsed === "object" && "@type" in parsed ? parsed["@type"] : void 0 }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A,C" }) }).catch(() => {
+    });
     collectCandidates(parsed, candidates);
   });
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/jsonld.ts:22", message: "JSON-LD candidates", data: { candidateCount: candidates.length, candidateTypes: candidates.map((c) => c["@type"]) }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A,C" }) }).catch(() => {
+  });
   return candidates[0] ?? null;
 }
 function collectCandidates(payload, bucket) {
@@ -1853,13 +1897,123 @@ function findPropertyValue($, context, prop) {
   return normalizeText(node.attr("content")) || normalizeText(node.attr("href")) || normalizeText(node.attr("src")) || normalizeText(node.text());
 }
 
+// src/scraper/extractors/browser.ts
+var SIMPLE_PROPS2 = ["name", "description", "image", "recipeYield", "prepTime", "cookTime", "totalTime"];
+function extractRecipeBrowser(html) {
+  const jsonLdRecipe = extractJsonLdBrowser(html);
+  if (jsonLdRecipe) {
+    return { recipe: jsonLdRecipe, source: "jsonld" };
+  }
+  const microdataRecipe = extractMicrodataBrowser(html);
+  if (microdataRecipe) {
+    return { recipe: microdataRecipe, source: "microdata" };
+  }
+  return { recipe: null, source: null };
+}
+function extractJsonLdBrowser(html) {
+  if (typeof globalThis.DOMParser === "undefined") {
+    return null;
+  }
+  const parser = new globalThis.DOMParser();
+  const doc = parser.parseFromString(html, "text/html");
+  const scripts = doc.querySelectorAll('script[type="application/ld+json"]');
+  const candidates = [];
+  scripts.forEach((script) => {
+    const content = script.textContent;
+    if (!content) return;
+    const parsed = safeJsonParse(content);
+    if (!parsed) return;
+    collectCandidates2(parsed, candidates);
+  });
+  return candidates[0] ?? null;
+}
+function extractMicrodataBrowser(html) {
+  if (typeof globalThis.DOMParser === "undefined") {
+    return null;
+  }
+  const parser = new globalThis.DOMParser();
+  const doc = parser.parseFromString(html, "text/html");
+  const recipeEl = doc.querySelector('[itemscope][itemtype*="schema.org/Recipe"]');
+  if (!recipeEl) {
+    return null;
+  }
+  const recipe = {
+    "@type": "Recipe"
+  };
+  SIMPLE_PROPS2.forEach((prop) => {
+    const value = findPropertyValue2(recipeEl, prop);
+    if (value) {
+      recipe[prop] = value;
+    }
+  });
+  const ingredients = [];
+  recipeEl.querySelectorAll('[itemprop="recipeIngredient"]').forEach((el) => {
+    const text = normalizeText(
+      el.getAttribute("content") || el.textContent || void 0
+    );
+    if (text) ingredients.push(text);
+  });
+  if (ingredients.length) {
+    recipe.recipeIngredient = ingredients;
+  }
+  const instructions = [];
+  recipeEl.querySelectorAll('[itemprop="recipeInstructions"]').forEach((el) => {
+    const text = normalizeText(el.getAttribute("content")) || normalizeText(el.querySelector('[itemprop="text"]')?.textContent || void 0) || normalizeText(el.textContent || void 0);
+    if (text) instructions.push(text);
+  });
+  if (instructions.length) {
+    recipe.recipeInstructions = instructions;
+  }
+  if (recipe.name || ingredients.length) {
+    return recipe;
+  }
+  return null;
+}
+function findPropertyValue2(context, prop) {
+  const node = context.querySelector(`[itemprop="${prop}"]`);
+  if (!node) return void 0;
+  return normalizeText(node.getAttribute("content")) || normalizeText(node.getAttribute("href")) || normalizeText(node.getAttribute("src")) || normalizeText(node.textContent || void 0);
+}
+function collectCandidates2(payload, bucket) {
+  if (!payload) return;
+  if (Array.isArray(payload)) {
+    payload.forEach((entry) => collectCandidates2(entry, bucket));
+    return;
+  }
+  if (typeof payload !== "object") {
+    return;
+  }
+  if (isRecipeNode(payload)) {
+    bucket.push(payload);
+    return;
+  }
+  const graph = payload["@graph"];
+  if (Array.isArray(graph)) {
+    graph.forEach((entry) => collectCandidates2(entry, bucket));
+  }
+}
+
 // src/scraper/extractors/index.ts
+function isBrowser() {
+  try {
+    return typeof globalThis.DOMParser !== "undefined";
+  } catch {
+    return false;
+  }
+}
 function extractRecipe(html) {
+  if (isBrowser()) {
+    return extractRecipeBrowser(html);
+  }
   const jsonLdRecipe = extractJsonLd(html);
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/index.ts:6", message: "JSON-LD extraction result", data: { hasJsonLd: !!jsonLdRecipe }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "C,D" }) }).catch(() => {
+  });
   if (jsonLdRecipe) {
     return { recipe: jsonLdRecipe, source: "jsonld" };
   }
   const microdataRecipe = extractMicrodata(html);
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/extractors/index.ts:12", message: "Microdata extraction result", data: { hasMicrodata: !!microdataRecipe }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "D" }) }).catch(() => {
+  });
   if (microdataRecipe) {
     return { recipe: microdataRecipe, source: "microdata" };
   }
@@ -1868,12 +2022,20 @@ function extractRecipe(html) {
 
 // src/scraper/index.ts
 async function scrapeRecipe(url, options = {}) {
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/index.ts:7", message: "scrapeRecipe entry", data: { url, hasOptions: !!options }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A,B,C,D,E" }) }).catch(() => {
+  });
   const html = await fetchPage(url, options);
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/index.ts:9", message: "HTML fetched", data: { htmlLength: html?.length, htmlPreview: html?.substring(0, 200) }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "B" }) }).catch(() => {
+  });
   const { recipe } = extractRecipe(html);
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/index.ts:11", message: "extractRecipe result", data: { hasRecipe: !!recipe, recipeType: recipe?.["@type"], recipeName: recipe?.name }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A,C,D" }) }).catch(() => {
+  });
   if (!recipe) {
     throw new Error("No Schema.org recipe data found in page");
   }
   const soustackRecipe = fromSchemaOrg(recipe);
+  fetch("http://127.0.0.1:7243/ingest/7225c3b5-9ac2-4c94-b561-807ca9003b66", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ location: "scraper/index.ts:17", message: "fromSchemaOrg result", data: { hasSoustackRecipe: !!soustackRecipe, soustackRecipeName: soustackRecipe?.name }, timestamp: Date.now(), sessionId: "debug-session", runId: "run1", hypothesisId: "A" }) }).catch(() => {
+  });
   if (!soustackRecipe) {
     throw new Error("Schema.org data did not include a valid recipe");
   }