tracwell 0.3.0 → 0.4.0

package/README.md

@@ -257,3 +257,9 @@ database persistence. Queries/fragments are stripped; no IP, cookies or other
 request headers are sent. Raw UA is sent solely for collector classification and
 is not stored. Current provider matches are user-agent based, not IP verified.
 See the public `/docs/ai-crawlers` guide for setup and direct POST integration.
+
+Use 0.3.1 or newer on Cloudflare Workers: crawler delivery uses manual redirect
+handling because Workers does not support the fetch redirect error mode. Redirect
+responses are rejected without forwarding the server key.
+
+Version 0.4.0 expands server crawler recognition to 23 tokens across 12 providers. Existing server integrations must upgrade and redeploy to send newly supported crawlers; keys and browser installation are unchanged. The collector determines the purpose, including other for mixed/general crawling.

package/dist/npm/server.js

@@ -1 +1 @@
-var l=[{token:"ChatGPT-User",provider:"OpenAI",category:"answers"},{token:"OAI-SearchBot",provider:"OpenAI",category:"indexing"},{token:"GPTBot",provider:"OpenAI",category:"training"},{token:"Claude-User",provider:"Anthropic",category:"answers"},{token:"Claude-SearchBot",provider:"Anthropic",category:"indexing"},{token:"ClaudeBot",provider:"Anthropic",category:"training"},{token:"Perplexity-User",provider:"Perplexity",category:"answers"},{token:"PerplexityBot",provider:"Perplexity",category:"indexing"}];function p(n){return l.find(r=>new RegExp(`(?:^|[\\s;(])${r.token}(?:/|[\\s;)]|$)`,"i").test(n))}async function h(n,r){let c=n.headers.get("user-agent")??"";if(!["GET","HEAD"].includes(n.method)||!p(c))return{status:"ignored"};let o=new URL(n.url);if(/\.(?:js|css|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|mp4|map)$/i.test(o.pathname)||/^\/(?:api|_next)(?:\/|$)/.test(o.pathname))return{status:"ignored"};let a=crypto.randomUUID(),e;try{if(e=new URL(r.collectorUrl??"https://collect.tracwell.app/v1/server/crawlers"),e.username||e.password||e.search||e.hash||e.protocol!=="https:"&&!(e.protocol==="http:"&&["localhost","127.0.0.1","[::1]"].includes(e.hostname))||!/^tw_sk_[A-Za-z0-9_-]+$/.test(r.serverKey))throw new Error("Invalid configuration")}catch{return{status:"failed",eventId:a,code:"CONFIGURATION"}}o.search="",o.hash="",o.username="",o.password="";let d=JSON.stringify({schema_version:1,event_id:a,timestamp:new Date().toISOString(),url:o.href,user_agent:c.slice(0,1024),...r.statusCode===void 0?{}:{status_code:r.statusCode}});for(let i=0;i<3;i++){try{let s=await fetch(e,{method:"POST",redirect:"error",signal:AbortSignal.timeout(2e3),headers:{"content-type":"application/json",authorization:`Bearer ${r.serverKey}`},body:d});if(s.status===202){let t=await s.json();return t&&typeof t=="object"&&"batch_id"in t&&t.batch_id===a&&"accepted_events"in t&&t.accepted_events===1&&"receipt_version"in t&&t.receipt_version===1?{status:"accepted",eventId:a}:{status:"failed",eventId:a,code:"DELIVERY_REJECTED"}}if(s.status!==408&&s.status!==429&&s.status<500)return{status:"failed",eventId:a,code:"DELIVERY_REJECTED"}}catch{}i<2&&await new Promise(s=>setTimeout(s,150*(i+1)))}return{status:"failed",eventId:a,code:"DELIVERY_UNAVAILABLE"}}export{h as trackAiCrawler};
+var l=[{token:"ChatGPT-User",provider:"OpenAI",category:"answers"},{token:"OAI-SearchBot",provider:"OpenAI",category:"indexing"},{token:"GPTBot",provider:"OpenAI",category:"training"},{token:"Claude-User",provider:"Anthropic",category:"answers"},{token:"Claude-SearchBot",provider:"Anthropic",category:"indexing"},{token:"ClaudeBot",provider:"Anthropic",category:"training"},{token:"Perplexity-User",provider:"Perplexity",category:"answers"},{token:"PerplexityBot",provider:"Perplexity",category:"indexing"},{token:"MistralAI-User",provider:"Mistral",category:"answers"},{token:"MistralAI-Index",provider:"Mistral",category:"indexing"},{token:"MistralAI-Training",provider:"Mistral",category:"training"},{token:"Amzn-User",provider:"Amazon",category:"answers"},{token:"Amzn-SearchBot",provider:"Amazon",category:"indexing"},{token:"Amazonbot",provider:"Amazon",category:"other"},{token:"meta-externalfetcher",provider:"Meta",category:"answers"},{token:"meta-externalagent",provider:"Meta",category:"other"},{token:"FacebookBot",provider:"Meta",category:"other"},{token:"DuckAssistBot",provider:"DuckDuckGo",category:"answers"},{token:"Applebot",provider:"Apple",category:"other"},{token:"Bytespider",provider:"ByteDance",category:"other"},{token:"CCBot",provider:"Common Crawl",category:"other"},{token:"Google-CloudVertexBot",provider:"Google",category:"other"},{token:"AI2Bot",provider:"Ai2",category:"training"}];function p(i){return l.find(r=>new RegExp(`(?:^|[\\s;(])${r.token}(?:/|[\\s;)]|$)`,"i").test(i))}async function y(i,r){let c=i.headers.get("user-agent")??"";if(!["GET","HEAD"].includes(i.method)||!p(c))return{status:"ignored"};let o=new URL(i.url);if(/\.(?:js|css|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|mp4|map)$/i.test(o.pathname)||/^\/(?:api|_next)(?:\/|$)/.test(o.pathname))return{status:"ignored"};let a=crypto.randomUUID(),e;try{if(e=new URL(r.collectorUrl??"https://collect.tracwell.app/v1/server/crawlers"),e.username||e.password||e.search||e.hash||e.protocol!=="https:"&&!(e.protocol==="http:"&&["localhost","127.0.0.1","[::1]"].includes(e.hostname))||!/^tw_sk_[A-Za-z0-9_-]+$/.test(r.serverKey))throw new Error("Invalid configuration")}catch{return{status:"failed",eventId:a,code:"CONFIGURATION"}}o.search="",o.hash="",o.username="",o.password="";let d=JSON.stringify({schema_version:1,event_id:a,timestamp:new Date().toISOString(),url:o.href,user_agent:c.slice(0,1024),...r.statusCode===void 0?{}:{status_code:r.statusCode}});for(let s=0;s<3;s++){try{let n=await fetch(e,{method:"POST",redirect:"manual",signal:AbortSignal.timeout(2e3),headers:{"content-type":"application/json",authorization:`Bearer ${r.serverKey}`},body:d});if(n.status===202){let t=await n.json();return t&&typeof t=="object"&&"batch_id"in t&&t.batch_id===a&&"accepted_events"in t&&t.accepted_events===1&&"receipt_version"in t&&t.receipt_version===1?{status:"accepted",eventId:a}:{status:"failed",eventId:a,code:"DELIVERY_REJECTED"}}if(n.status!==408&&n.status!==429&&n.status<500)return{status:"failed",eventId:a,code:"DELIVERY_REJECTED"}}catch{}s<2&&await new Promise(n=>setTimeout(n,150*(s+1)))}return{status:"failed",eventId:a,code:"DELIVERY_UNAVAILABLE"}}export{y as trackAiCrawler};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tracwell",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Privacy-aware browser analytics and optional server-side AI crawler tracking for Tracwell",
   "license": "MIT",
   "type": "module",