x-summary 0.1.0

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "x-summary",
+  "version": "0.1.0",
+  "description": "Summarize X.com Following and For You feeds using Playwright and an LLM",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/barbieri/x-summary.git"
+  },
+  "type": "module",
+  "private": false,
+  "license": "GPL-3.0-or-later",
+  "engines": {
+    "node": ">=24.15.0"
+  },
+  "packageManager": "pnpm@11.2.2",
+  "files": [
+    "dist/bundle",
+    "schemas",
+    "config.example.json",
+    "INSTRUCTIONS.example.md",
+    ".env.example",
+    "LICENSE",
+    "README.md"
+  ],
+  "bin": {
+    "x-summary-scrape": "dist/bundle/scrape.mjs",
+    "x-summary-summarize": "dist/bundle/summarize.mjs"
+  },
+  "scripts": {
+    "build": "tsc && node scripts/build-cli.mjs",
+    "build:cli": "node scripts/build-cli.mjs",
+    "check": "biome check --error-on-warnings .",
+    "check:fix": "biome check --write --error-on-warnings .",
+    "format": "biome format --write .",
+    "lint": "biome lint --error-on-warnings .",
+    "qa": "run-p check build test typecheck",
+    "start": "run-s --silent scrape summarize --",
+    "start:bundle": "run-s --silent scrape:bundle summarize:bundle --",
+    "scrape": "node --import tsx src/scrape.ts",
+    "summarize": "node --import tsx src/summarize.ts",
+    "scrape:bundle": "node dist/bundle/scrape.mjs",
+    "summarize:bundle": "node dist/bundle/summarize.mjs",
+    "inspect:x": "node --import tsx scripts/inspect-x-dom.ts",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit -p tsconfig.json && tsc --noEmit -p tsconfig.test.json",
+    "prepare": "node scripts/prepare.mjs",
+    "prepublishOnly": "pnpm run build"
+  },
+  "dependencies": {
+    "@ai-sdk/anthropic": "^3.0.76",
+    "@ai-sdk/google": "^3.0.75",
+    "@ai-sdk/openai": "^3.0.64",
+    "@ai-sdk/xai": "^3.0.91",
+    "@openrouter/ai-sdk-provider": "^2.9.0",
+    "ai": "^6.0.185",
+    "ai-sdk-provider-opencode-sdk": "^3.0.2",
+    "ajv": "8.20.0",
+    "ajv-formats": "^3.0.1",
+    "dotenv": "^17.4.2",
+    "pino": "^10.3.1",
+    "playwright": "^1.60.0"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^2.4.15",
+    "@tsconfig/strictest": "^2.0.8",
+    "@types/node": "^25.9.1",
+    "esbuild": "^0.28.0",
+    "husky": "^9.1.7",
+    "npm-run-all": "^4.1.5",
+    "pino-pretty": "^13.1.3",
+    "tsx": "^4.20.6",
+    "typescript": "6.0.3",
+    "vitest": "^4.1.7"
+  }
+}

package/schemas/config.schema.json

@@ -0,0 +1,95 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://x-summary.local/schemas/config.schema.json",
+  "title": "X Summary Config",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["ownerHandle", "timeWindowMinutes", "instructionsPath", "monitored", "llm"],
+  "properties": {
+    "ownerHandle": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Handle of the logged-in owner; used to verify the active browser session."
+    },
+    "timeWindowMinutes": {
+      "type": "integer",
+      "minimum": 1,
+      "description": "Collect posts from the last N minutes."
+    },
+    "statePath": {
+      "type": "string",
+      "default": "./tmp/state.json",
+      "description": "Path to the persisted scrape state JSON file."
+    },
+    "instructionsPath": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Path to INSTRUCTIONS.md prepended to the LLM prompt."
+    },
+    "monitored": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1
+      },
+      "uniqueItems": true,
+      "description": "Handles not followed but included in summarization."
+    },
+    "headless": {
+      "type": "boolean",
+      "default": true,
+      "description": "When true (default), run the browser in headless mode. When false, run the browser in visible mode."
+    },
+    "abortOnIncorrectOwnerHandle": {
+      "type": "boolean",
+      "default": false,
+      "description": "When true, exit with an error if login is required or the active session does not match ownerHandle. When false (default), keep the browser open and wait for you to log in or switch accounts."
+    },
+    "browserProfilePath": {
+      "type": "string",
+      "default": "./tmp/browser-profile",
+      "description": "Chrome user-data directory reused across runs (cookies, localStorage, IndexedDB). Not used for Vitest."
+    },
+    "browserCdpEndpoint": {
+      "type": "string",
+      "description": "Optional Chrome DevTools Protocol URL (e.g. http://127.0.0.1:9222). Attach to Chrome you start manually — best for avoiding FedCM/Google login failures."
+    },
+    "timezone": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Optional IANA timezone for the summarization prompt (e.g. America/Sao_Paulo). Used when humanizing timestamps and the covered time window."
+    },
+    "parallelTabs": {
+      "type": "integer",
+      "minimum": 1,
+      "default": 4,
+      "description": "Number of browser tabs used in parallel when scraping post detail pages and referenced posts."
+    },
+    "summarizeNoPosts": {
+      "type": "boolean",
+      "default": false,
+      "description": "When false (default), summarization skips the LLM and returns an empty string if the state has no posts. When true, call the LLM anyway so INSTRUCTIONS.md can shape a custom no-posts message (e.g. translation or tone)."
+    },
+    "llm": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["provider", "model"],
+      "properties": {
+        "provider": {
+          "type": "string",
+          "enum": ["openai", "anthropic", "google", "xai", "openrouter", "opencode"]
+        },
+        "model": {
+          "type": "string",
+          "minLength": 1
+        },
+        "temperature": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Sampling temperature passed to generateText. Use lower values for more deterministic and focused summaries."
+        }
+      }
+    }
+  }
+}

package/schemas/state.schema.json

@@ -0,0 +1,123 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://x-summary.local/schemas/state.schema.json",
+  "title": "X Summary State",
+  "description": "Scraped timeline snapshot. Feed lists hold ordered post hrefs; full post data (stats, author, timestamp, body, references, thread) lives in posts keyed by canonical href.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "timestamp",
+    "cutoffTimestamp",
+    "posts",
+    "following",
+    "forYouSuggestions",
+    "monitored"
+  ],
+  "properties": {
+    "timestamp": {
+      "description": "ISO8601 time when this state snapshot was generated.",
+      "type": "string",
+      "format": "date-time"
+    },
+    "cutoffTimestamp": {
+      "description": "Absolute ISO8601 instant for the start of the collection window (not a duration or minute offset). First run: scrape time minus timeWindowMinutes. Incremental run: previous state timestamp.",
+      "type": "string",
+      "format": "date-time"
+    },
+    "posts": {
+      "description": "All scraped posts keyed by canonical href. following, forYouSuggestions, monitored, and post references/thread point into this map.",
+      "type": "object",
+      "propertyNames": { "$ref": "#/$defs/postHref" },
+      "additionalProperties": { "$ref": "#/$defs/post" }
+    },
+    "following": {
+      "description": "Ordered hrefs into posts for Following > Recent.",
+      "type": "array",
+      "items": { "$ref": "#/$defs/postHref" }
+    },
+    "forYouSuggestions": {
+      "description": "Ordered hrefs into posts for For You suggestions.",
+      "type": "array",
+      "items": { "$ref": "#/$defs/postHref" }
+    },
+    "monitored": {
+      "description": "Ordered hrefs into posts per monitored handle.",
+      "type": "object",
+      "propertyNames": {
+        "type": "string",
+        "minLength": 1
+      },
+      "additionalProperties": {
+        "type": "array",
+        "items": { "$ref": "#/$defs/postHref" }
+      }
+    }
+  },
+  "$defs": {
+    "postHref": {
+      "description": "Canonical post URL; must exist as a key in posts.",
+      "type": "string",
+      "format": "uri"
+    },
+    "stats": {
+      "description": "Engagement counts at scrape time.",
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["comments", "reposts", "likes"],
+      "properties": {
+        "comments": { "type": "integer", "minimum": 0 },
+        "reposts": { "type": "integer", "minimum": 0 },
+        "likes": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "resolvedLink": {
+      "description": "External URL after redirects, with page metadata.",
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["url"],
+      "properties": {
+        "url": { "type": "string", "format": "uri" },
+        "title": { "type": "string" },
+        "description": { "type": "string" }
+      }
+    },
+    "post": {
+      "description": "Post payload in posts (key = canonical href). references and thread are href lists into posts.",
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["stats"],
+      "properties": {
+        "author": {
+          "description": "Handle of the post author.",
+          "type": "string",
+          "minLength": 1
+        },
+        "timestamp": {
+          "description": "ISO8601 time when the post was published.",
+          "type": "string",
+          "format": "date-time"
+        },
+        "stats": { "$ref": "#/$defs/stats" },
+        "body": {
+          "description": "Post text as markdown. Omitted for reposts without custom text.",
+          "type": "string"
+        },
+        "thread": {
+          "description": "Hrefs of ancestor posts in the same thread (root-first); keys in posts.",
+          "type": "array",
+          "items": { "$ref": "#/$defs/postHref" }
+        },
+        "links": {
+          "description": "External links from the body, resolved with title and description.",
+          "type": "array",
+          "items": { "$ref": "#/$defs/resolvedLink" }
+        },
+        "references": {
+          "description": "Hrefs of quoted or referenced posts; keys in posts.",
+          "type": "array",
+          "items": { "$ref": "#/$defs/postHref" }
+        }
+      }
+    }
+  }
+}