personal-agent-benchmark 0.0.0-stage → 0.2.0

package/package.json

@@ -1,6 +1,123 @@
 {
   "name": "personal-agent-benchmark",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.2.0",
+  "description": "A real-world benchmark for personal agents: tasks on real sites, replayed offline from network recordings.",
+  "license": "UNLICENSED",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/maxgent-ai/Personal-Agent-Benchmark.git"
+  },
+  "homepage": "https://github.com/maxgent-ai/Personal-Agent-Benchmark#readme",
+  "bugs": "https://github.com/maxgent-ai/Personal-Agent-Benchmark/issues",
+  "bin": {
+    "pab": "bin/pab.js"
+  },
+  "exports": {
+    "./package.json": "./package.json",
+    "./browser": {
+      "types": "./dist/src/browser.d.ts",
+      "default": "./dist/src/browser.js"
+    },
+    "./cli": {
+      "types": "./dist/src/cli.d.ts",
+      "default": "./dist/src/cli.js"
+    },
+    "./grading": {
+      "types": "./dist/src/grading.d.ts",
+      "default": "./dist/src/grading.js"
+    },
+    "./judge": {
+      "types": "./dist/src/judge.d.ts",
+      "default": "./dist/src/judge.js"
+    },
+    "./scoring": {
+      "types": "./dist/src/scoring.d.ts",
+      "default": "./dist/src/scoring.js"
+    },
+    "./tasks": {
+      "types": "./dist/src/tasks.d.ts",
+      "default": "./dist/src/tasks.js"
+    },
+    "./replay/engine": {
+      "types": "./dist/src/replay/engine.d.ts",
+      "default": "./dist/src/replay/engine.js"
+    },
+    "./replay/launcher": {
+      "types": "./dist/src/replay/launcher.d.ts",
+      "default": "./dist/src/replay/launcher.js"
+    },
+    "./core/archive": {
+      "types": "./dist/src/core/archive.d.ts",
+      "default": "./dist/src/core/archive.js"
+    },
+    "./core/audit": {
+      "types": "./dist/src/core/audit.d.ts",
+      "default": "./dist/src/core/audit.js"
+    },
+    "./core/bundle": {
+      "types": "./dist/src/core/bundle.d.ts",
+      "default": "./dist/src/core/bundle.js"
+    },
+    "./core/lookup": {
+      "types": "./dist/src/core/lookup.d.ts",
+      "default": "./dist/src/core/lookup.js"
+    },
+    "./core/normalize": {
+      "types": "./dist/src/core/normalize.d.ts",
+      "default": "./dist/src/core/normalize.js"
+    },
+    "./core/rules": {
+      "types": "./dist/src/core/rules.d.ts",
+      "default": "./dist/src/core/rules.js"
+    },
+    "./core/store": {
+      "types": "./dist/src/core/store.d.ts",
+      "default": "./dist/src/core/store.js"
+    },
+    "./core/types": {
+      "types": "./dist/src/core/types.d.ts",
+      "default": "./dist/src/core/types.js"
+    },
+    "./core/util": {
+      "types": "./dist/src/core/util.d.ts",
+      "default": "./dist/src/core/util.js"
+    },
+    "./core/warc": {
+      "types": "./dist/src/core/warc.d.ts",
+      "default": "./dist/src/core/warc.js"
+    },
+    "./core/writer": {
+      "types": "./dist/src/core/writer.d.ts",
+      "default": "./dist/src/core/writer.js"
+    }
+  },
+  "files": [
+    "bin",
+    "dist/src",
+    "docs",
+    "plugins",
+    "sites"
+  ],
+  "scripts": {
+    "build": "rm -rf dist && tsc -p tsconfig.json",
+    "prepare": "npm run build",
+    "test": "npm run build && node --test \"dist/test/**/*.test.js\"",
+    "pab": "node bin/pab.js"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^1.3.0",
+    "cheerio": "^1.2.0",
+    "playwright-core": "^1.64.1",
+    "tar": "^7.5.22",
+    "warcio": "^2.4.12",
+    "whistle": "^2.10.12",
+    "yaml": "^2.9.1"
+  },
+  "devDependencies": {
+    "@types/node": "^26.6.5",
+    "typescript": "^7.0.2"
+  }
+}

package/plugins/whistle.bench-replay/index.js

@@ -0,0 +1 @@
+module.exports = require('../../dist/src/replay/whistle-plugin.js');

package/plugins/whistle.bench-replay/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "whistle.bench-replay",
+  "version": "0.1.0",
+  "description": "Replays a Personal Agent Benchmark recording; never forwards requests to the network",
+  "main": "index.js",
+  "whistleConfig": {}
+}

package/sites/_default.yaml

@@ -0,0 +1,80 @@
+# Rules applied to every site. Site files add to these lists.
+site: _default
+hosts: []
+drop_params:
+  - _
+  - cb
+  - cachebuster
+  - rnd
+  - random
+  - nocache
+  - t
+  - ts
+  - timestamp
+  - utm_*
+  - fbclid
+  - gclid
+  - msclkid
+  - "*session*"
+body_drop:
+  - csrf*
+  - "*csrf*"
+  - "*token*"
+  - nonce
+  - timestamp
+  - requestId
+  - request_id
+  - "*session*"
+exclude:
+  - "*/errors/validateCaptcha*"
+# Pages that show what was done before them (cart, checkout): replay answers them with the last
+# recording, which is where the task left them, instead of the first.
+latest:
+  - "*/cart*"
+  - "*/basket*"
+  - "*/bag"
+  - "*/bag/*"
+  - "*/trolley*"
+  - "*/checkout*"
+block:
+  - "*.google-analytics.com"
+  - "*.googletagmanager.com"
+  - "*.doubleclick.net"
+  - "*.googlesyndication.com"
+  - "*.facebook.net"
+  - "*.facebook.com"
+  - "*.hotjar.com"
+  - "*.scorecardresearch.com"
+  - "*.adsrvr.org"
+  - "*.criteo.com"
+  - "*.criteo.net"
+  - "*.bing.com"
+  - "*.clarity.ms"
+  # ad-tech cookie syncing and impression pixels
+  - "*.agkn.com"
+  - "*.zeotap.com"
+  - "*.samplicio.us"
+  - "*.samba.tv"
+  - "*.mookie1.com"
+  - "*.pubmatic.com"
+  - "*.smartadserver.com"
+  - "*.fwmrm.net"
+  - "*.mediarithmics.com"
+  - "*.adnxs.com"
+  - "*.rubiconproject.com"
+  - "*.openx.net"
+  - "*.casalemedia.com"
+  - "*.bidswitch.net"
+  - "*.3lift.com"
+  - "*.rlcdn.com"
+  - "*.demdex.net"
+  # Chrome's own background services, not page traffic
+  - "content-autofill.googleapis.com"
+  - "update.googleapis.com"
+  - "optimizationguide-pa.googleapis.com"
+  - "safebrowsing.googleapis.com"
+  - "android.clients.google.com"
+  - "clients2.google.com"
+  - "accounts.google.com/ListAccounts*"
+  - "www.gstatic.com/ohttp_gateway/**"
+  - "www.google.com/async/folae*"

package/sites/amazon.com.yaml

@@ -0,0 +1,125 @@
+# Amazon (US).
+site: amazon
+hosts:
+  - "amazon.com"
+  - "*.amazon.com"
+  - "*.media-amazon.com"
+  - "*.ssl-images-amazon.com"
+  - "*.amazon-adsystem.com"
+drop_params:
+  - ref
+  - ref_
+  - qid
+  - sr
+  - crid
+  - sprefix
+  - pd_rd_*
+  - pf_rd_*
+  - content-id
+  - _encoding
+  - dib
+  - dib_tag
+  - hv*
+  - tag
+  - linkCode
+  - sp_csd
+  - spIA
+  - spLa
+  - sbo
+  - cv_ct_cx
+  - __mk_*
+  - rnid
+  - smid
+  - th
+  - psc
+  - showmri
+  - rshVal
+  - stamp
+  - request-id
+exclude:
+  - "www.amazon.com/checkpoint/**"
+  - "www.amazon.com/errors/validateCaptcha*"
+block:
+  - "*.amazon-adsystem.com"
+  - "fls-na.amazon.com"
+  - "unagi.amazon.com"
+  - "unagi-na.amazon.com"
+  - "www.amazon.com/1/batch/**"
+  - "www.amazon.com/uedata*"
+  - "www.amazon.com/rd/uedata*"
+  - "www.amazon.com/empty.gif*"
+  - "*.amazon.com/csm/**"
+  - "www.amazon.com/af/feedback-link*"
+  - "aax-*.amazon.com"
+  - "sponsored-ads.amazon.com"
+  - "www.amazon.com/rd/uedata*"
+endpoints:
+  # A product page is determined by its ASIN; title slugs, ref and tracking parameters vary.
+  - name: product
+    match: "GET /*/dp/{asin}/**"
+    key: "/dp/{asin}"
+    keep_params: []
+  - name: product_dp
+    match: "GET /dp/{asin}/**"
+    key: "/dp/{asin}"
+    keep_params: []
+  - name: product_gp
+    match: "GET /gp/product/{asin}/**"
+    key: "/dp/{asin}"
+    keep_params: []
+  - name: search
+    match: "GET /s"
+    host: "www.amazon.com"
+    keep_params: [k, i, page, rh, s]
+    kind: search
+    text_param: k
+  - name: search_legacy               # the header search form posts to /s/ref=...?field-keywords=
+    match: "GET /s/**"
+    host: "www.amazon.com"
+    key: "/s"
+    keep_params: [field-keywords, page]
+    kind: search
+    text_param: field-keywords
+  - name: suggestions
+    match: "GET /api/2017/suggestions"
+    host: "completion.amazon.com"
+    keep_params: [prefix, mid, alias]
+    kind: search
+    text_param: prefix
+    similarity: 0.3
+  - name: suggestions_www
+    match: "GET /suggestions"
+    host: "www.amazon.com"
+    keep_params: [prefix, alias]
+    kind: search
+    text_param: prefix
+    similarity: 0.3
+  # Cart: only what was done while recording replays; another item or quantity misses. Cart
+  # pages answer with their last recording (latest in _default.yaml).
+  - name: add_to_cart
+    match: "POST /cart/add-to-cart/**"
+    kind: stateful
+    body_keep: [ASIN, quantity]
+  - name: cart_update_quantity
+    match: "POST /cart/ref=ox_sc_update_quantity*"
+    kind: stateful
+    body_keep: [asin, actionType]
+  # Page components fetched by JS. Bodies carry the session id from cookies, which the
+  # bundle stores redacted, so match on the fields that select the content.
+  - name: cross_border_interstitial
+    match: "POST /cross_border_interstitial_sp/render"
+    body_keep: [url, pageType]
+  # page-type is filled in by page script and reads "undefined" when a component loads before
+  # the page has set it, so the same request is recorded with "undefined" and replayed with
+  # "Detail"; the component path and body decide the content.
+  - name: ajax_components
+    match: "POST /acp/**"
+    drop_params: [page-type]
+    fuzzy: true
+  - name: video_player
+    match: "POST /vap/**"
+    fuzzy: true
+  - name: search_pagination
+    match: "POST /s/query"
+    keep_params: [k, page]
+    fuzzy: true