webrecipe 0.1.2 → 0.1.3

package/dist/src/benchmark/ground-truth.js

@@ -1,5 +1,13 @@
 import { DATASET, PAGE_SIZE, search } from '../../fixtures/data.js';
-const identity = (item) => String(item.url ?? '');
+const identity = (item) => {
+    const url = String(item.url ?? '');
+    try {
+        return new URL(url).pathname;
+    }
+    catch {
+        return url;
+    }
+};
 const KNOWN = new Set(DATASET.map((r) => `/item/${r.id}`));
 const PAGES = Math.ceil(DATASET.length / PAGE_SIZE);
 /** Every match, not just page one: a different ranking is not a wrong answer. */

package/dist/src/compiler/html.js

@@ -101,7 +101,7 @@ function compileFrom(trace, plan, source) {
     const untemplated = required.find((name) => !templated.has(name));
     if (untemplated !== undefined)
         return { refused: `required input "${untemplated}" not templated` };
-    const items = extractHtmlItems(draft, body);
+    const items = extractHtmlItems(draft, body, source.url);
     if (!verifyAgainstBrowser(trace, plan, items).equivalent) {
         return { refused: equivalenceRefusal(items.length, browserItemsOf(trace, plan).length) };
     }

package/dist/src/compiler/verify.js

@@ -10,7 +10,7 @@ import { diffGolden } from '../benchmark/golden.js';
  */
 /** What the browser extracted from this visit, using the plan's own selectors. */
 export function browserItemsOf(trace, plan) {
-    return extractBySelector(trace.finalHtml, plan.itemSelector, plan.fields);
+    return extractBySelector(trace.finalHtml, plan.itemSelector, plan.fields, trace.finalUrl);
 }
 export function verifyAgainstBrowser(trace, plan, recipeItems) {
     const browserItems = browserItemsOf(trace, plan);

package/dist/src/executor/extract.js

@@ -29,10 +29,10 @@ export function extractJsonItems(recipe, payload) {
             : renderRowTemplate(spec, row).trim(),
     ])));
 }
-export function extractHtmlItems(recipe, html) {
+export function extractHtmlItems(recipe, html, pageUrl) {
     if (recipe.output.type !== 'html')
         throw new Error('extractHtmlItems requires an html recipe');
-    return extractBySelector(html, recipe.output.items.selector, recipe.output.items.fields);
+    return extractBySelector(html, recipe.output.items.selector, recipe.output.items.fields, pageUrl);
 }
 /**
  * The structural signature of an HTML response: which of the recipe's declared
@@ -90,16 +90,20 @@ export function parseHtmlFragment(html) {
     const wrapper = FRAGMENT_WRAPPERS.find(([pattern]) => pattern.test(head));
     return cheerio.load(wrapper ? wrapper[1](head) : html);
 }
+/** Attributes holding a URL, reported absolute the way the browser's `a.href` reports them. */
+const URL_ATTRIBUTES = new Set(['href', 'src']);
 /** One interpretation of a field spec, shared by cheerio and the browser. */
 export function planFields(fields) {
     return Object.entries(fields).map(([name, spec]) => {
         if (spec === '')
             return { name, mode: 'own-text' };
-        if (spec.startsWith('@'))
-            return { name, mode: 'own-attr', attribute: spec.slice(1) };
+        if (spec.startsWith('@')) {
+            const attribute = spec.slice(1);
+            return { name, mode: 'own-attr', attribute, url: URL_ATTRIBUTES.has(attribute) };
+        }
         const suffix = attributeSuffix(spec);
         return suffix
-            ? { name, mode: 'find-attr', selector: suffix.selector, attribute: suffix.attribute }
+            ? { name, mode: 'find-attr', ...suffix, url: URL_ATTRIBUTES.has(suffix.attribute) }
             : { name, mode: 'find-text', selector: spec };
     });
 }
@@ -136,15 +140,39 @@ function attributeSuffix(spec) {
         return null;
     return { selector, attribute: match[1] };
 }
+/** Resolves a URL attribute as the browser does; an empty or unparsable value is left alone. */
+function absolute(value, base) {
+    if (value.trim() === '')
+        return value;
+    try {
+        return new URL(value, base).href;
+    }
+    catch {
+        return value;
+    }
+}
+/** Makes href and src fields absolute against the document's base URL, in place. */
+export function resolveUrlFields(items, plans, base) {
+    const urlFields = plans.filter((f) => (f.mode === 'own-attr' || f.mode === 'find-attr') && f.url).map((f) => f.name);
+    for (const item of items) {
+        for (const name of urlFields) {
+            const value = item[name];
+            if (typeof value === 'string')
+                item[name] = absolute(value, base);
+        }
+    }
+    return items;
+}
 /**
  * Extracts items from HTML with a bare selector and field map, independent of a
  * recipe. Used to reconstruct what the browser saw so a candidate recipe can be
- * checked against it.
+ * checked against it. Given the page's URL, href and src come back absolute;
+ * without it they come back as written.
  */
-export function extractBySelector(html, selector, fields) {
+export function extractBySelector(html, selector, fields, pageUrl) {
     const $ = parseHtmlFragment(html);
     const plans = planFields(fields);
-    return $(selector).toArray().map((element) => {
+    const items = $(selector).toArray().map((element) => {
         const item = $(element);
         const row = {};
         // Field candidates ask for `a`, `a@href` and `a@id` of the same item; one find serves all three.
@@ -176,4 +204,8 @@ export function extractBySelector(html, selector, fields) {
         }
         return row;
     });
+    if (pageUrl === undefined)
+        return items;
+    const baseHref = $('base[href]').first().attr('href');
+    return resolveUrlFields(items, plans, baseHref === undefined ? pageUrl : absolute(baseHref, pageUrl));
 }

package/dist/src/executor/strategies/browser.js

@@ -1,7 +1,7 @@
 import { measureResult } from '../../measurement.js';
 import { openSession } from '../../browser/session.js';
 import { navigateAndSettle, guardPage } from '../../browser/navigate.js';
-import { planFields } from '../extract.js';
+import { planFields, resolveUrlFields } from '../extract.js';
 import { countTokens } from '../tokens.js';
 import { emptyMeta } from '../../types.js';
 export const BROWSER_PLANS = {};
@@ -43,6 +43,8 @@ export class BrowserStrategy {
                     default: return [f.name, el.querySelector(f.selector)?.textContent?.trim() ?? null];
                 }
             }))), planFields(plan.fields)));
+            // Resolved in node rather than in the page, by the same rule the http path uses.
+            resolveUrlFields(items, planFields(plan.fields), await session.page.evaluate(() => document.baseURI));
             meta.pageNavigations = session.cost.pageNavigations;
             meta.networkRequests = session.cost.networkRequests;
             meta.bytesDownloaded = session.cost.bytesDownloaded;

package/dist/src/executor/strategies/http-html.js

@@ -35,7 +35,7 @@ export class HttpHtmlStrategy {
         if (res.status !== recipe.validation.status) {
             return { items: [], meta, status: res.status, payload: undefined };
         }
-        const items = extractHtmlItems(recipe, res.body);
+        const items = extractHtmlItems(recipe, res.body, res.url);
         meta.llmTokens = countTokens(formatItems(items, 'tsv'));
         return { items, meta, payload: htmlSignature(recipe, items), status: res.status };
     }

package/dist/src/executor/strategies/warm-browser.js

@@ -1,7 +1,7 @@
 import { measureResult } from '../../measurement.js';
 import { BrowserPool } from '../../browser/pool.js';
 import { navigateAndSettle, guardPage } from '../../browser/navigate.js';
-import { planFields } from '../extract.js';
+import { planFields, resolveUrlFields } from '../extract.js';
 import { countTokens } from '../tokens.js';
 import { BROWSER_PLANS } from './browser.js';
 import { emptyMeta } from '../../types.js';
@@ -38,6 +38,8 @@ export class WarmBrowserStrategy {
                     default: return [f.name, el.querySelector(f.selector)?.textContent?.trim() ?? null];
                 }
             }))), planFields(plan.fields)));
+            // Resolved in node rather than in the page, by the same rule the http path uses.
+            resolveUrlFields(items, planFields(plan.fields), await warm.page.evaluate(() => document.baseURI));
             // Zero only when the pool already had a browser. The first call starts
             // Chromium like any cold run does, and reporting none said otherwise.
             meta.browserLaunches = warm.launched ? 1 : 0;

package/dist/src/net/politeness.js

@@ -101,7 +101,7 @@ export class PolitenessLayer {
                 continue;
             }
             return {
-                status: res.status, headers: Object.fromEntries(res.headers.entries()),
+                status: res.status, headers: Object.fromEntries(res.headers.entries()), url: current,
                 body: buffer.toString('utf8'), bytesDownloaded: buffer.byteLength, waitedMs: 0,
             };
         }

package/dist/src/recorder/index.js

@@ -142,6 +142,7 @@ export async function record(plan, task, sites, allowed) {
             actions,
             requests: pending,
             finalHtml,
+            finalUrl: session.page.url(),
             ...(session.page.viewportSize() === null ? {} : { viewport: session.page.viewportSize() }),
             cost: session.cost,
             dispose: () => bodies.dispose(),

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "webrecipe",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "type": "module",
   "engines": {
     "node": ">=22"
   },
   "bin": {
-    "webrecipe": "./dist/src/cli.js"
+    "webrecipe": "dist/src/cli.js"
   },
   "scripts": {
     "build": "tsc -p tsconfig.build.json",
@@ -41,7 +41,7 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/Pillsoon/webrecipe.git"
+    "url": "git+https://github.com/Pillsoon/webrecipe.git"
   },
   "keywords": [
     "web",