tracwell 0.2.0 → 0.3.1

package/README.md

@@ -13,12 +13,33 @@ The production browser artifact is served from the existing Collector domain:
   src="https://collect.tracwell.app/script.js"
   data-project-key="tw_live_..."
   data-collection-mode="private"
-  data-consent="granted"
-  data-respect-do-not-track="true"
 ></script>
 ```
 
-Loading the script captures the initial page view and exposes the same client
+### Optional script attributes
+
+| Attribute | Values | Default |
+| --- | --- | --- |
+| `data-project-key` | Your public project key (never a server key) | Required |
+| `data-collection-mode` | `private`, `product` | `private` |
+| `data-consent` | `granted`, `required`, `denied` | `granted` |
+| `data-respect-do-not-track` | `true`, `false` | `true` |
+
+Keep collection mode explicit and aligned with website settings. The generated
+snippet omits default consent and Do Not Track attributes. Add
+`data-consent="required"` to wait for consent before identity creation or event
+collection. After the script loads, forward the visitor's saved choice and
+subsequent changes using `window.tracwell?.setConsent("granted")` or
+`window.tracwell?.setConsent("denied")`. An early call before the script loads
+has no effect; initialize the consent integration after SDK load.
+
+Only these four data attributes are supported. Use the npm SDK for other
+initialization options, including batching, persistence overrides, and a custom
+collector URL. The script reads attributes at load time and does not fetch
+configuration from the dashboard; update the installed snippet when changing
+website settings.
+
+When consent and Do Not Track permit collection, loading the script captures the initial page view and exposes the same client
 as `window.tracwell` for custom events:
 
 ```js
@@ -85,8 +106,9 @@ batching, retries, consent, and delivery remain in this core package.
 - Retry bounded transient failures without changing event IDs.
 - Respect configured consent and Do Not Track behavior before collection.
 
-The SDK must remain browser-only, small, non-blocking, and unable to set trusted
-collector fields. It must not contain secrets or connect to ClickHouse.
+The default SDK entry must remain browser-only, small, non-blocking, and unable
+to set trusted collector fields. It must not contain secrets or connect to
+ClickHouse. The separate server entry accepts a server-side key as documented below.
 
 ## Identity and attribution
 
@@ -213,3 +235,29 @@ pnpm --filter {packages/sdk} test
 pnpm --filter {packages/sdk} build:cdn
 pnpm --filter {packages/sdk} check:package
 ```
+
+## Server-only AI crawler entry
+
+`tracwell/server` exports `trackAiCrawler(request, { serverKey, statusCode?,
+collectorUrl? })`. Import only on your backend; never expose `tw_sk_`
+keys in browser code. This separate bundle adds nothing to the default browser
+entry or CDN script. Available starting with npm package version 0.3.0. The unchanged browser
+transport continues reporting browser SDK version 0.2.0.
+
+Call from Next.js proxy/middleware or another server/edge handler. Pass the
+promise to the hosting runtime's `waitUntil` (and inspect its result), or await
+on a persistent server. Keep existing request handling intact. The helper ignores
+ordinary traffic, API paths and common static assets. Robots, sitemap and Markdown
+requests remain trackable. Handler/cache placement determines coverage.
+
+Results are `ignored`, `accepted` (matching Queue receipt), or `failed` with a
+non-sensitive error code. Three total attempts use the same event ID, timestamp
+and sanitized URL. Retries are in memory only. Do not treat acceptance as confirmed
+database persistence. Queries/fragments are stripped; no IP, cookies or other
+request headers are sent. Raw UA is sent solely for collector classification and
+is not stored. Current provider matches are user-agent based, not IP verified.
+See the public `/docs/ai-crawlers` guide for setup and direct POST integration.
+
+Use 0.3.1 or newer on Cloudflare Workers: crawler delivery uses manual redirect
+handling because Workers does not support the fetch redirect error mode. Redirect
+responses are rejected without forwarding the server key.

package/dist/npm/server.d.ts

@@ -0,0 +1,18 @@
+export interface CrawlerTrackingOptions {
+    /** Secret tw_sk_ key. Never include in browser code. */
+    serverKey: string;
+    collectorUrl?: string;
+    statusCode?: number;
+}
+export type CrawlerTrackingResult = {
+    status: "ignored";
+} | {
+    status: "accepted";
+    eventId: string;
+} | {
+    status: "failed";
+    eventId: string;
+    code: "CONFIGURATION" | "DELIVERY_REJECTED" | "DELIVERY_UNAVAILABLE";
+};
+/** Await on a persistent server, or pass the promise to the runtime's waitUntil. */
+export declare function trackAiCrawler(request: Request, options: CrawlerTrackingOptions): Promise<CrawlerTrackingResult>;

package/dist/npm/server.js

@@ -0,0 +1 @@
+var l=[{token:"ChatGPT-User",provider:"OpenAI",category:"answers"},{token:"OAI-SearchBot",provider:"OpenAI",category:"indexing"},{token:"GPTBot",provider:"OpenAI",category:"training"},{token:"Claude-User",provider:"Anthropic",category:"answers"},{token:"Claude-SearchBot",provider:"Anthropic",category:"indexing"},{token:"ClaudeBot",provider:"Anthropic",category:"training"},{token:"Perplexity-User",provider:"Perplexity",category:"answers"},{token:"PerplexityBot",provider:"Perplexity",category:"indexing"}];function p(s){return l.find(r=>new RegExp(`(?:^|[\\s;(])${r.token}(?:/|[\\s;)]|$)`,"i").test(s))}async function h(s,r){let c=s.headers.get("user-agent")??"";if(!["GET","HEAD"].includes(s.method)||!p(c))return{status:"ignored"};let o=new URL(s.url);if(/\.(?:js|css|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|mp4|map)$/i.test(o.pathname)||/^\/(?:api|_next)(?:\/|$)/.test(o.pathname))return{status:"ignored"};let a=crypto.randomUUID(),e;try{if(e=new URL(r.collectorUrl??"https://collect.tracwell.app/v1/server/crawlers"),e.username||e.password||e.search||e.hash||e.protocol!=="https:"&&!(e.protocol==="http:"&&["localhost","127.0.0.1","[::1]"].includes(e.hostname))||!/^tw_sk_[A-Za-z0-9_-]+$/.test(r.serverKey))throw new Error("Invalid configuration")}catch{return{status:"failed",eventId:a,code:"CONFIGURATION"}}o.search="",o.hash="",o.username="",o.password="";let d=JSON.stringify({schema_version:1,event_id:a,timestamp:new Date().toISOString(),url:o.href,user_agent:c.slice(0,1024),...r.statusCode===void 0?{}:{status_code:r.statusCode}});for(let i=0;i<3;i++){try{let n=await fetch(e,{method:"POST",redirect:"manual",signal:AbortSignal.timeout(2e3),headers:{"content-type":"application/json",authorization:`Bearer ${r.serverKey}`},body:d});if(n.status===202){let t=await n.json();return t&&typeof t=="object"&&"batch_id"in t&&t.batch_id===a&&"accepted_events"in t&&t.accepted_events===1&&"receipt_version"in t&&t.receipt_version===1?{status:"accepted",eventId:a}:{status:"failed",eventId:a,code:"DELIVERY_REJECTED"}}if(n.status!==408&&n.status!==429&&n.status<500)return{status:"failed",eventId:a,code:"DELIVERY_REJECTED"}}catch{}i<2&&await new Promise(n=>setTimeout(n,150*(i+1)))}return{status:"failed",eventId:a,code:"DELIVERY_UNAVAILABLE"}}export{h as trackAiCrawler};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "tracwell",
-  "version": "0.2.0",
-  "description": "Small, privacy-aware browser analytics SDK for Tracwell",
+  "version": "0.3.1",
+  "description": "Privacy-aware browser analytics and optional server-side AI crawler tracking for Tracwell",
   "license": "MIT",
   "type": "module",
   "sideEffects": false,
@@ -13,6 +13,11 @@
       "types": "./dist/npm/index.d.ts",
       "import": "./dist/npm/index.js",
       "default": "./dist/npm/index.js"
+    },
+    "./server": {
+      "types": "./dist/npm/server.d.ts",
+      "import": "./dist/npm/server.js",
+      "default": "./dist/npm/server.js"
     }
   },
   "files": [
@@ -21,7 +26,9 @@
     "dist/npm/config.d.ts",
     "dist/npm/types.d.ts",
     "LICENSE",
-    "README.md"
+    "README.md",
+    "dist/npm/server.js",
+    "dist/npm/server.d.ts"
   ],
   "keywords": [
     "analytics",