twittertools-mcp 0.0.0-stage → 1.0.1

package/README.md

@@ -1,3 +1,68 @@
-# Temporary Holding Version
+# twittertools-mcp
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+[Model Context Protocol](https://modelcontextprotocol.io) server for
+[TwitterTools](https://twittertools.com) — brings the toolkit to AI agents
+(Claude, ChatGPT, Cursor, ZCode, …). No login, no tracking, no API keys.
+
+## Tools
+
+| Tool | What it does |
+| --- | --- |
+| `get_tweet` | Fetch a public post by URL or ID: text, author, media with direct CDN URLs |
+| `get_thread` | Unroll a thread into its posts, in order (partial results are flagged) |
+| `count_chars` | Measure text against X's real 280-character limit (CJK ×2, links = 23) |
+| `split_thread` | Split long text into numbered, limit-fitting posts |
+| `parse_tweet_url` | URL/ID → ID, author handle, posting time, canonical permalink |
+
+`get_tweet` / `get_thread` call the stateless [twittertools.com](https://twittertools.com)
+API, which resolves public posts through X's free syndication endpoint. The other
+three run entirely offline. Media is **not** proxied: tweet results carry the
+direct `pbs.twimg.com` / `video.twimg.com` URLs, which any client can fetch on its own.
+
+## Install
+
+```sh
+# Claude Code
+claude mcp add twittertools -- npx -y twittertools-mcp
+```
+
+Or add it to any client that reads an `mcpServers` config
+(Claude Desktop's `claude_desktop_config.json`, Cursor's `.cursor/mcp.json`, …):
+
+```json
+{
+  "mcpServers": {
+    "twittertools": {
+      "command": "npx",
+      "args": ["-y", "twittertools-mcp"]
+    }
+  }
+}
+```
+
+Requires Node 18+.
+
+## Self-hosting
+
+Point `TWITTERTOOLS_API_BASE` at your own instance of
+[`server/downloader_server.py`](../server/) to keep lookups entirely on your
+infrastructure:
+
+```json
+{ "command": "npx", "args": ["-y", "twittertools-mcp"], "env": { "TWITTERTOOLS_API_BASE": "https://your-instance.example" } }
+```
+
+## Privacy
+
+The server is stateless: nothing is stored, no telemetry, no accounts. It talks
+only to the configured API base. The API itself is rate-limited per IP and keeps
+only aggregate, anonymous counters — see [twittertools.com/privacy](https://twittertools.com/privacy).
+
+## Development
+
+```sh
+npm install
+cd .. && npx vitest run mcp   # unit + stdio protocol tests (offline, mock API)
+```
+
+MIT — same as the [main project](https://github.com/steley/twittertools).

package/package.json

@@ -1,6 +1,40 @@
 {
   "name": "twittertools-mcp",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.0.1",
+  "mcpName": "io.github.steley/twittertools-mcp",
+  "description": "MCP server for twittertools.com — fetch X (Twitter) posts and threads, count and split posts with X's real rules. No login, no tracking.",
+  "license": "MIT",
+  "type": "module",
+  "bin": {
+    "twittertools-mcp": "server.js"
+  },
+  "files": [
+    "server.js",
+    "xrules.js"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "twitter",
+    "x",
+    "tweet",
+    "thread",
+    "twittertools"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/steley/twittertools.git",
+    "directory": "mcp"
+  },
+  "homepage": "https://twittertools.com",
+  "bugs": "https://github.com/steley/twittertools/issues",
+  "scripts": {
+    "start": "node server.js"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.0"
+  }
+}

package/server.js

@@ -0,0 +1,206 @@
+#!/usr/bin/env node
+/**
+ * TwitterTools MCP server (stdio) — brings twittertools.com to AI agents.
+ *
+ * Tools backed by the public API (stateless, no keys, rate-limited per IP):
+ *   get_tweet    -> GET {base}/api/tweet?id=...
+ *   get_thread   -> GET {base}/api/thread?url=...
+ * Local tools (offline, same engine as the website):
+ *   count_chars, split_thread, parse_tweet_url
+ *
+ * Media is NOT proxied through the API here: get_tweet results carry the
+ * direct pbs.twimg.com / video.twimg.com URLs, which any client can fetch.
+ *
+ * Set TWITTERTOOLS_API_BASE to point at a self-hosted instance.
+ */
+
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { ListToolsRequestSchema, CallToolRequestSchema } from "@modelcontextprotocol/sdk/types.js";
+import {
+  countTweet,
+  splitThread,
+  parseTweetInput,
+  snowflakeToDate,
+  permalinkFor,
+} from "./xrules.js";
+
+const NAME = "twittertools";
+const VERSION = "1.0.1"; // keep in sync with package.json
+const API_BASE = (process.env.TWITTERTOOLS_API_BASE || "https://twittertools.com").replace(/\/+$/, "");
+const ATTRIBUTION = "\n\nvia twittertools.com";
+const TEXT_INPUT_MAX = 100_000; // generous, but caps local work per call
+
+const urlOrIdSchema = {
+  type: "object",
+  properties: {
+    url_or_id: {
+      type: "string",
+      description: "Post URL (x.com or twitter.com), bare numeric ID, or text containing one",
+    },
+  },
+  required: ["url_or_id"],
+};
+
+const TOOLS = [
+  {
+    name: "get_tweet",
+    description:
+      "Fetch one public X (Twitter) post by URL or ID: text, author, engagement counts, " +
+      "and media with direct CDN URLs (photos, or MP4 video variants sorted by quality). " +
+      "Deleted, protected, or age-restricted posts are not available without an X login.",
+    inputSchema: urlOrIdSchema,
+  },
+  {
+    name: "get_thread",
+    description:
+      "Unroll a public X thread into its posts in order. Expensive upstream — call sparingly, " +
+      "and prefer pasting the thread's LAST post, which returns the whole chain. Results may " +
+      "be partial; the response says so when they are.",
+    inputSchema: urlOrIdSchema,
+  },
+  {
+    name: "count_chars",
+    description:
+      "Count text against X's real 280-character limit: CJK and emoji weigh 2, any link " +
+      "weighs exactly 23. Same engine as twittertools.com/tweet-character-counter/.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        text: { type: "string", description: "The text to measure" },
+      },
+      required: ["text"],
+    },
+  },
+  {
+    name: "split_thread",
+    description:
+      "Split long text into numbered posts (1/, 2/, …) that each fit X's 280 weighted-character " +
+      "limit, preferring sentence boundaries and never breaking a URL. Same engine as " +
+      "twittertools.com/tweet-splitter/.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        text: { type: "string", description: "The text to split" },
+        numbering: { type: "boolean", description: "Prefix each post with 'n/ ' (default true)" },
+      },
+      required: ["text"],
+    },
+  },
+  {
+    name: "parse_tweet_url",
+    description:
+      "Parse a post URL, bare ID, or free text containing an ID: returns the numeric ID, " +
+      "author handle, posting timestamp (decoded from the Snowflake ID), and canonical permalink. Offline.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        input: { type: "string", description: "URL, ID, or text containing either" },
+      },
+      required: ["input"],
+    },
+  },
+];
+
+function textResult(text, { error = false } = {}) {
+  return { content: [{ type: "text", text }], isError: error };
+}
+
+function strArg(args, field, max = 2000) {
+  const v = args?.[field];
+  if (typeof v !== "string" || !v.trim()) {
+    throw new Error(`Missing required argument: ${field} (string)`);
+  }
+  if (v.length > max) {
+    throw new Error(`${field} is too long (max ${max} characters)`);
+  }
+  return v;
+}
+
+async function callApi(endpoint, params) {
+  let res;
+  try {
+    res = await fetch(`${API_BASE}/api/${endpoint}?${new URLSearchParams(params)}`, {
+      headers: { accept: "application/json" },
+      signal: AbortSignal.timeout(20_000),
+    });
+  } catch (e) {
+    const timedOut = e?.name === "TimeoutError" || e?.name === "AbortError";
+    throw new Error(
+      timedOut
+        ? `twittertools API timed out after 20s (${API_BASE})`
+        : `twittertools API unreachable (${API_BASE}) — check the network or TWITTERTOOLS_API_BASE`
+    );
+  }
+  let body = null;
+  try {
+    body = await res.json();
+  } catch {
+    // non-JSON body (proxy error page, HTML) — fall through to the status message
+  }
+  if (!res.ok) {
+    throw new Error(body?.error || `twittertools API returned HTTP ${res.status}`);
+  }
+  return body;
+}
+
+async function callTool(name, args) {
+  switch (name) {
+    case "get_tweet": {
+      const data = await callApi("tweet", { id: strArg(args, "url_or_id") });
+      if (!data?.tweet) throw new Error("twittertools API returned no tweet");
+      return textResult(JSON.stringify(data.tweet, null, 2) + ATTRIBUTION);
+    }
+    case "get_thread": {
+      const data = await callApi("thread", { url: strArg(args, "url_or_id") });
+      if (!data?.tweets) throw new Error("twittertools API returned no thread");
+      const head = data?.partial
+        ? `Note: this thread result is PARTIAL (${data.reason || "incomplete"}) — posts may be missing.\n\n`
+        : "";
+      return textResult(head + JSON.stringify({ count: data.tweets.length, tweets: data.tweets }, null, 2) + ATTRIBUTION);
+    }
+    case "count_chars":
+      return textResult(JSON.stringify(countTweet(strArg(args, "text", TEXT_INPUT_MAX)), null, 2));
+    case "split_thread": {
+      const numbering = args?.numbering !== false;
+      const posts = splitThread(strArg(args, "text", TEXT_INPUT_MAX), numbering);
+      return textResult(JSON.stringify({ count: posts.length, posts }, null, 2));
+    }
+    case "parse_tweet_url": {
+      const parsed = parseTweetInput(strArg(args, "input"));
+      if (!parsed) {
+        return textResult("No post URL or ID found in the input.", { error: true });
+      }
+      const created = snowflakeToDate(parsed.id);
+      return textResult(
+        JSON.stringify(
+          {
+            id: parsed.id,
+            screenName: parsed.screenName,
+            createdAt: Number.isNaN(created.getTime()) ? null : created.toISOString(),
+            permalink: permalinkFor(parsed.id, parsed.screenName),
+          },
+          null,
+          2
+        )
+      );
+    }
+    default:
+      return textResult(`Unknown tool: ${name}`, { error: true });
+  }
+}
+
+const server = new Server({ name: NAME, version: VERSION }, { capabilities: { tools: {} } });
+
+server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: TOOLS }));
+
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  const { name, arguments: args } = request.params;
+  try {
+    return await callTool(name, args);
+  } catch (e) {
+    return textResult(`Error: ${e?.message || String(e)}`, { error: true });
+  }
+});
+
+await server.connect(new StdioServerTransport());

package/xrules.js

@@ -0,0 +1,241 @@
+/**
+ * X/Twitter text rules, ported 1:1 from the repo's TypeScript originals
+ * (src/lib/tweet-count.ts, tweet-split.ts, snowflake.ts) so this package can
+ * ship standalone. The repo vitest suite pins the originals; the mcp tests
+ * re-run the same cases against this port — keep the two in sync.
+ */
+
+// --------------------------------------------------------------------------- //
+// Weighted counting (twitter-text confit.json v3): most Latin/ASCII code
+// points weigh 1, everything else (CJK, emoji, most non-Latin scripts)
+// weighs 2, and any URL counts as exactly 23 (t.co wrapping). Limit 280.
+// --------------------------------------------------------------------------- //
+
+const WEIGHT_100_RANGES = [
+  [0x0000, 0x10ff],
+  [0x2000, 0x200d],
+  [0x2010, 0x201f],
+  [0x2032, 0x2037],
+];
+
+/** http(s) and www. links — unambiguous. */
+const URL_REGEX = /(https?:\/\/[^\s<>"']+|www\.[^\s<>"']+)/gi;
+/**
+ * Scheme-less domains (X counts "x.com/foo" as a link too). The real
+ * twitter-text regex validates every IANA TLD; this approximation covers the
+ * common ones, which is accurate for virtually all real posts.
+ */
+const BARE_DOMAIN_REGEX =
+  /\b(?:[a-z0-9-]+\.)+(?:com|org|net|io|co|dev|app|xyz|me|gov|edu|info|cn|jp|uk|de|fr|ru|br|in|nl|it|es|se|no|fi|ca|au|us)(?:\/[^\s<>"']*)?/gi;
+/** X never includes trailing punctuation in a link. */
+const TRAILING_PUNCT = /[.,;:!?)\]}'"。，！？）、》…]+$/;
+
+const MAX_WEIGHTED = 28_000; // 280 chars x 100
+
+function isWeight100(cp) {
+  return WEIGHT_100_RANGES.some(([lo, hi]) => cp >= lo && cp <= hi);
+}
+
+export function extractUrls(text) {
+  const primary = text.match(URL_REGEX) ?? [];
+  // mask the unambiguous matches, then count scheme-less domains in what's
+  // left so no URL is counted twice
+  const masked = text.replace(URL_REGEX, (m) => '\u0000'.repeat(m.length));
+  const bare = masked.match(BARE_DOMAIN_REGEX) ?? [];
+  return [...primary, ...bare].map((u) => u.replace(TRAILING_PUNCT, '')).filter(Boolean);
+}
+
+export function countTweet(text) {
+  const urls = extractUrls(text);
+  let withoutUrls = text;
+  for (const u of urls) withoutUrls = withoutUrls.replace(u, '');
+
+  let units = urls.length * 2_300; // each link counts as 23 chars
+  let codePoints = 0;
+  let heavyCodePoints = 0;
+
+  for (const ch of withoutUrls) {
+    codePoints++;
+    const cp = ch.codePointAt(0) ?? 0;
+    if (isWeight100(cp)) {
+      units += 100;
+    } else {
+      units += 200;
+      heavyCodePoints++;
+    }
+  }
+
+  const weightedLength = Math.round(units / 100);
+  return {
+    weightedLength,
+    codePoints,
+    heavyCodePoints,
+    urlCount: urls.length,
+    limit: 280,
+    remaining: 280 - weightedLength,
+    valid: units <= MAX_WEIGHTED,
+  };
+}
+
+// --------------------------------------------------------------------------- //
+// Thread splitting at the weighted limit, preferring sentence boundaries
+// over line breaks over word breaks. Links are atomic.
+// --------------------------------------------------------------------------- //
+
+const LIMIT = 280;
+/** Reserved for the "12/ " prefix when numbering is on (covers up to 999 parts). */
+const NUMBERING_RESERVE = 5;
+
+const SENTENCE_END = new Set(['.', '!', '?', '…', '。', '！', '？']);
+const CLOSERS = new Set(['.', '!', '?', '…', '。', '！', '？', ',', ';', ':', '"', "'", ')', ']', '」', '』', '”', '’']);
+
+/** Break a line into sentences; decimals ("3.5") and abbreviations without a
+ * following space don't count as boundaries. */
+function sentencesOf(line) {
+  const out = [];
+  const chars = Array.from(line);
+  let start = 0;
+  for (let i = 0; i < chars.length; i++) {
+    if (!SENTENCE_END.has(chars[i])) continue;
+    let j = i + 1;
+    while (j < chars.length && CLOSERS.has(chars[j])) j++;
+    if (j >= chars.length || chars[j] === ' ') {
+      const s = chars.slice(start, j).join('').trim();
+      if (s) out.push(s);
+      while (j < chars.length && chars[j] === ' ') j++;
+      start = j;
+      i = j - 1;
+    } else {
+      i = j - 1; // "3.5", "x.com" — resume scanning after the punctuation run
+    }
+  }
+  const rest = chars.slice(start).join('').trim();
+  if (rest) out.push(rest);
+  return out;
+}
+
+/** Split an oversized piece on whitespace; a single unbreakable word longer
+ * than the limit gets cut by code points (URLs keep working — they carry no
+ * spaces, and their 23-char weight is what counts, not their raw length). */
+function hardSplit(s, eff) {
+  const out = [];
+  let cur = '';
+  for (const word of s.split(/\s+/)) {
+    const pieces =
+      countTweet(word).weightedLength > eff
+        ? chunkCodePoints(word, eff)
+        : [word];
+    for (const piece of pieces) {
+      const cand = cur ? cur + ' ' + piece : piece;
+      if (countTweet(cand).weightedLength <= eff) cur = cand;
+      else {
+        if (cur) out.push(cur);
+        cur = piece;
+      }
+    }
+  }
+  if (cur) out.push(cur);
+  return out;
+}
+
+function chunkCodePoints(s, eff) {
+  const out = [];
+  let cur = '';
+  for (const ch of Array.from(s)) {
+    const cand = cur + ch;
+    if (countTweet(cand).weightedLength <= eff) cur = cand;
+    else {
+      out.push(cur);
+      cur = ch;
+    }
+  }
+  if (cur) out.push(cur);
+  return out;
+}
+
+function segment(text, eff) {
+  const segs = [];
+  let pendingGlue = '';
+  for (const rawLine of text.replace(/\r\n?/g, '\n').split('\n')) {
+    const line = rawLine.trim();
+    if (!line) {
+      pendingGlue = '\n\n'; // paragraph break
+      continue;
+    }
+    for (const sentence of sentencesOf(line)) {
+      const pieces =
+        countTweet(sentence).weightedLength > eff ? hardSplit(sentence, eff) : [sentence];
+      pieces.forEach((piece, i) => {
+        segs.push({ text: piece, glue: segs.length === 0 ? '' : i === 0 ? pendingGlue || ' ' : ' ' });
+      });
+      pendingGlue = '';
+    }
+    pendingGlue = pendingGlue || '\n'; // next line joins with a line break
+  }
+  return segs;
+}
+
+/** Split `text` into posts that each fit X's 280-character weighted limit.
+ * With `numbering`, every post gets an "n/ " prefix and the usable budget is
+ * reserved for it. Returns the ready-to-post texts. */
+export function splitThread(text, numbering = true) {
+  const input = text.trim();
+  if (!input) return [];
+  const eff = LIMIT - (numbering ? NUMBERING_RESERVE : 0);
+  const parts = [];
+  let cur = '';
+  for (const seg of segment(input, eff)) {
+    const cand = cur ? cur + seg.glue + seg.text : seg.text;
+    if (countTweet(cand).weightedLength <= eff) cur = cand;
+    else {
+      if (cur.trim()) parts.push(cur.trim());
+      cur = seg.text;
+    }
+  }
+  if (cur.trim()) parts.push(cur.trim());
+  if (!numbering) return parts;
+  return parts.map((p, i) => `${i + 1}/ ${p}`);
+}
+
+// --------------------------------------------------------------------------- //
+// Snowflake IDs: the top 41 bits are milliseconds since X's custom epoch.
+// --------------------------------------------------------------------------- //
+
+export const X_EPOCH = 1288834974657;
+
+export function snowflakeToDate(id) {
+  const n = typeof id === 'bigint' ? id : BigInt(id);
+  const ms = Number(n >> 22n) + X_EPOCH;
+  // IDs past X's real range decode beyond the max JS Date (±8.64e15 ms) —
+  // surface them as Invalid Date instead of a silently-wrong value
+  return new Date(ms > 8.64e15 || ms < -8.64e15 ? NaN : ms);
+}
+
+// The (?:^|[^A-Za-z0-9-]) boundary keeps lookalike hosts (notx.com, foo-x.com)
+// from matching as x.com — the host must start at a word boundary.
+const STATUS_RE =
+  /(?:^|[^A-Za-z0-9-])(?:https?:\/\/)?(?:www\.|mobile\.)?(?:x|twitter)\.com\/([A-Za-z0-9_]{1,15})\/status(?:es)?\/(\d{5,25})/i;
+const BARE_ID_RE = /^\d{5,25}$/;
+
+/** Extract a status id (and screen name when present) from a URL, bare ID, or free text. */
+export function parseTweetInput(input) {
+  const text = (input ?? '').trim();
+  if (!text) return null;
+
+  const m = text.match(STATUS_RE);
+  if (m) return { id: m[2], screenName: m[1].toLowerCase() === 'i' ? null : m[1] };
+
+  if (BARE_ID_RE.test(text)) return { id: text, screenName: null };
+
+  // A bare ID hidden inside longer text
+  const idMatch = text.match(/(\d{15,25})/);
+  if (idMatch) return { id: idMatch[1], screenName: null };
+
+  return null;
+}
+
+export function permalinkFor(id, screenName) {
+  return screenName
+    ? `https://x.com/${screenName}/status/${id}`
+    : `https://x.com/i/web/status/${id}`;
+}