wiki-plugin-hitchhiker 0.3.1 → 0.4.0

package/README.md

@@ -39,24 +39,20 @@ surface. It installs:
 - **Page-flag drop target** — drop a URL / link / page-flag onto a flag: the favicon shows
   a loud pulsing highlight; on drop it opens a ghost **preview card** (see the server below).
 
-## 3. Server component (`server/server.js`, ESM)
-`GET /system/unfurl.json?url=…` → a normalized **static** link-preview card.
-Since 0.3.0 the two extractor libraries are **imported on first request, not at
-start-up** — they cost about 14 MB of heap together, and a host that never
-unfurls should never pay it. Start-up cost of this server is now under 1 MB.
-`{ url, title, description, provider, author, type, image, favicon, source }`.
-- oEmbed metadata first (`@extractus/oembed-extractor`), OpenGraph/Twitter/JSON-LD fallback
-  (`open-graph-scraper`). Metadata only — never the oEmbed HTML embed (no live third-party
-  iframes/scripts).
-- The thumbnail is fetched **server-side and inlined as a `data:` URI**, so the card makes
-  zero client→provider requests.
-- **SSRF-guarded**: public `http`/`https` only; private/loopback/link-local ranges and
-  `*.localhost`/`.local`/`.internal` refused before any fetch. 30-min in-memory cache.
-  Never throws — returns a minimal fallback card offline/on error.
+## 3. Server component (`server/server.js`, ESM) — two tiny routes, no dependencies
+- `GET /system/farm-domains.json` → `{ local: [...], public: [...] }` — which domains this farm serves.
+- `GET /system/outposts.json` → `{ unfurl: ["http://mini.private.fish", …] }` — which **Outposts** the
+  browser may ask for link previews. Read from `{status}/outpost.json` per site (wins), else the
+  `OUTPOST_URL` env (comma-separated), else `[]`. Cached by mtime; no restart to change.
 
-> ⚠️ **Private-farm only for now.** Before any public-farm deployment, harden the SSRF guard
-> against redirect-following (the extractor libs do their own fetch and may follow redirects
-> to internal hosts) and tighten image caps.
+**Since 0.4.0 this plugin does no unfurling.** The link-preview builder — the two heavy extractor
+libraries, ~14 MB of heap — is `wiki-plugin-unfurl`, installed on one well-resourced host (an
+Outpost). `wiki-ux.js` calls that host **directly from the browser** (its route sets
+`Access-Control-Allow-Origin: *`; a plain GET is a simple request, no preflight), shuffling the
+list and trying each with a 2 s timeout. No Outpost, or none answering, means the drop report
+renders without a card. Nothing logs. Absence is a normal state.
+
+The list shape is deliberate: swapping the source for a DNS-SD SRV lookup later is one function.
 
 ## Install / build
 ```sh

package/client/wiki-ux.js

@@ -1145,14 +1145,44 @@
 
   // Ask our own server to unfurl a URL into a static preview card (no third-party
   // calls from the browser; the thumbnail comes back inlined as a data URI).
+  // Link previews come from an Outpost — a host running wiki-plugin-unfurl —
+  // called straight from the browser. This farm only tells us which ones exist
+  // (GET /system/outposts.json, a list). We shuffle and try each in turn with a
+  // short timeout; the first card wins; every failure is swallowed. No Outpost,
+  // or none answering, means null — and renderDropReport already draws a plain
+  // report for a null card. Absence is normal, not an error.
+  var _outposts = null
+  function outposts() {
+    if (_outposts) return _outposts
+    _outposts = fetch(location.origin + '/system/outposts.json')
+      .then(function (r) { return r.ok ? r.json() : {} })
+      .then(function (j) { return (j && Array.isArray(j.unfurl) ? j.unfurl : []).filter(Boolean) })
+      .catch(function () { return [] })
+    return _outposts
+  }
+  function shuffle(a) {
+    var b = a.slice()
+    for (var i = b.length - 1; i > 0; i--) { var j = Math.floor(Math.random() * (i + 1)); var t = b[i]; b[i] = b[j]; b[j] = t }
+    return b
+  }
   function unfurl(url) {
-    return fetch(location.origin + '/system/unfurl.json?url=' + encodeURIComponent(url))
-      .then(function (r) {
-        return r.ok ? r.json() : null
-      })
-      .catch(function () {
-        return null
-      })
+    return outposts().then(function (list) {
+      var order = shuffle(list)
+      var tryNext = function (i) {
+        if (i >= order.length) return null
+        var ctl = typeof AbortController === 'function' ? new AbortController() : null
+        var timer = ctl && setTimeout(function () { ctl.abort() }, 2000)
+        return fetch(order[i] + '/system/unfurl.json?url=' + encodeURIComponent(url), ctl ? { signal: ctl.signal } : {})
+          .then(function (r) { return r.ok ? r.json() : null })
+          .catch(function () { return null })
+          .then(function (card) {
+            if (timer) clearTimeout(timer)
+            if (card && typeof card === 'object' && 'title' in card) { card.outpost = order[i]; return card }
+            return tryNext(i + 1)
+          })
+      }
+      return tryNext(0)
+    })
   }
 
   function cardHtml(card) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "wiki-plugin-hitchhiker",
-  "version": "0.3.1",
-  "description": "House UX + default-pages plugin for the hitchhiker farms: ships default pages (Tools, Changes to this Site) and hosts the wiki-ux.js client override (friendlier factory chooser + page-flag tool page), loaded globally by the wiki-security-hitchhiker bootstrap.",
+  "version": "0.4.0",
+  "description": "House UX + default-pages plugin for the hitchhiker farms: ships default pages (Tools, Changes to this Site), hosts the wiki-ux.js client override, and tells the browser which Outposts serve link previews. No dependencies; the heavy half is wiki-plugin-unfurl.",
   "type": "module",
   "files": [
     "factory.json",
@@ -20,10 +20,6 @@
     "url": "https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-hitchhiker/issues"
   },
   "homepage": "https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-hitchhiker#readme",
-  "dependencies": {
-    "@extractus/oembed-extractor": "^4.1.0",
-    "open-graph-scraper": "^6.12.0"
-  },
   "fedwiki": {
     "api": {
       "version": 1,

package/server/server.js

@@ -1,50 +1,25 @@
-// wiki-plugin-hitchhiker — server component (ESM)
+// wiki-plugin-hitchhiker — server component. ESM.
 //
-// The farm runs Node v24 / wiki 0.40+, which loads plugin servers via import()
-// — so this matches wiki-plugin-similarity's ESM server (no server/package.json).
-// NOTE: for eventual publish to an older/public farm, revisit portability — an ESM
-// server silently 404s on wiki-server that uses require() + Node < 22.12.
+// Two small routes and nothing else:
+//   GET /system/farm-domains.json — which domains this farm serves (local/public)
+//   GET /system/outposts.json     — which Outposts the browser may ask for services
 //
-// GET /system/unfurl.json?url=<url>
-//   Link-preview "unfurl" for a dropped URL. Returns a normalized STATIC card:
-//     { url, title, description, provider, author, type, image, favicon, source }
-//   - oEmbed metadata first (@extractus/oembed-extractor), OpenGraph/Twitter/JSON-LD
-//     fallback (open-graph-scraper). We use metadata only — never the oEmbed html
-//     embed — so nothing live/third-party is injected (static-cards-only).
-//   - The thumbnail is fetched server-side and inlined as a data: URI, so the card
-//     makes ZERO client→provider requests when the report renders.
-//   - SSRF-guarded (public http/https only). In-memory TTL cache. Never throws to
-//     the client: on any failure it returns a minimal card so a drop always works.
+// Since 0.4.0 this plugin does no unfurling and carries no dependencies. Link
+// previews are built by wiki-plugin-unfurl on an Outpost host, and the browser
+// (client/wiki-ux.js) calls that host directly. This server's whole job for
+// unfurl is to hand the browser the list.
 //
-// v1 scope: private farm (owner-only). Before public use, harden SSRF against
-// redirect-following: since 0.3.1 the OpenGraph and image fetches walk redirects
-// by hand and re-check every hop (safeFetch); the oEmbed endpoint hop remains
-// the library's own — see fromOembed.
+// Discovery is a LIST of base URLs, tried in random order by the client —
+// client-side round-robin, no registry:
+//   {status}/outpost.json  { "unfurl": ["http://mini.private.fish", ...] }  per site, wins
+//   OUTPOST_URL            comma-separated                                 farm default
+// The list shape is deliberate: readOutposts() is the one function that changes
+// when the source becomes a DNS-SD SRV lookup. Do not hardcode a single URL.
 
-import dns from 'node:dns/promises'
-import net from 'node:net'
 import fs from 'node:fs'
 import path from 'node:path'
 import os from 'node:os'
 
-// The two extractor libraries are imported on first use, not at start-up.
-// Together they load about 14 MB of heap — eighty per cent of the memory of
-// every plugin server on the farm combined — and this server is resident in
-// the shared process on every site of the host, whether or not anything ever
-// asks for a link preview. Lazy import means a host that never unfurls never
-// pays. Each import is awaited once and cached, so the second request costs
-// nothing; a failed import is not cached, so a transient error does not stick.
-let ogsP = null
-const ogs = (...a) => (ogsP ||= import('open-graph-scraper').then(m => m.default, e => { ogsP = null; throw e })).then(f => f(...a))
-let oembedP = null
-const oembedExtract = (...a) =>
-  (oembedP ||= import('@extractus/oembed-extractor').then(m => m.extract, e => { oembedP = null; throw e })).then(f => f(...a))
-
-const CACHE = new Map()
-const TTL = 1000 * 60 * 30 // 30 minutes
-const FETCH_MS = 8000
-const MAX_IMAGE_BYTES = 300 * 1024
-
 // ── farm-domain listing ("is this host ours?") ────────────────────────────────
 // primary farm root = 'local' (the *.localhost private farm); extra roots = 'public'
 // (the Nextcloud mirror). A directory is a wiki domain if it contains a pages/ dir.
@@ -87,198 +62,46 @@ const listFarmDomains = primaryRoot => {
   return out
 }
 
-// ── SSRF guard ────────────────────────────────────────────────────────────────
-const isPrivateIp = ip => {
-  if (net.isIP(ip) === 4) {
-    const [a, b] = ip.split('.').map(Number)
-    return (
-      a === 0 ||
-      a === 127 ||
-      a === 10 ||
-      (a === 172 && b >= 16 && b <= 31) ||
-      (a === 192 && b === 168) ||
-      (a === 169 && b === 254) ||
-      a >= 224
-    )
-  }
-  const s = ip.toLowerCase()
-  return s === '::1' || s === '::' || s.startsWith('fc') || s.startsWith('fd') || s.startsWith('fe80')
-}
 
-const assertPublicUrl = async raw => {
-  const u = new URL(raw) // throws on bad url
-  if (u.protocol !== 'http:' && u.protocol !== 'https:') throw new Error('protocol not allowed')
-  const host = u.hostname.toLowerCase()
-  if (host === 'localhost' || host.endsWith('.localhost') || host.endsWith('.local') || host.endsWith('.internal'))
-    throw new Error('internal host blocked')
-  // Literal-IP hosts are range-checked directly; names are resolved first.
-  const addrs = net.isIP(host) ? [{ address: host }] : await dns.lookup(host, { all: true })
-  for (const { address } of addrs) if (isPrivateIp(address)) throw new Error('private address blocked')
-  return u
-}
-
-// ── a fetch that re-checks every hop ─────────────────────────────────────────
-// redirect:'follow' would let a public host 302 us onto 127.0.0.1 after the
-// guard has already passed the starting URL. So follow by hand: manual
-// redirects, assertPublicUrl on each Location, and a small hop cap.
-const MAX_HOPS = 4
-const safeFetch = async (raw, init = {}) => {
-  let url = raw
-  for (let hop = 0; hop <= MAX_HOPS; hop++) {
-    await assertPublicUrl(url)
-    const r = await fetch(url, { ...init, redirect: 'manual' })
-    if (r.status >= 300 && r.status < 400) {
-      const loc = r.headers.get('location')
-      if (!loc) return r
-      url = new URL(loc, url).href
-      continue
-    }
-    return r
-  }
-  throw new Error('too many redirects')
-}
-
-export const __safeFetchForTest = (...a) => safeFetch(...a)
-
-// ── thumbnail → data URI (no client→provider request when the card renders) ───
-const inlineImage = async imgUrl => {
-  if (!imgUrl) return ''
-  try {
-    const r = await safeFetch(imgUrl, { signal: AbortSignal.timeout(FETCH_MS) })
-    if (!r.ok) return ''
-    const type = r.headers.get('content-type') || ''
-    if (!/^image\//i.test(type)) return ''
-    const len = Number(r.headers.get('content-length') || 0)
-    if (len && len > MAX_IMAGE_BYTES) return ''
-    const buf = Buffer.from(await r.arrayBuffer())
-    if (buf.length > MAX_IMAGE_BYTES) return ''
-    return `data:${type.split(';')[0]};base64,${buf.toString('base64')}`
-  } catch {
-    return ''
-  }
-}
-
-// ── extractors ────────────────────────────────────────────────────────────────
-const fromOembed = async url => {
-  try {
-    // Residual: the extractor discovers an oEmbed endpoint from the page and
-    // fetches it with its own client. The starting URL is checked by the route,
-    // the thumbnail it returns is re-checked by inlineImage, but the endpoint
-    // hop itself is the library's. Acceptable on a host that only unfurls
-    // public URLs; not acceptable as the only guard. Hence the redirect-checked
-    // path for OpenGraph and the image, which are the two fetches we control.
-    const d = await oembedExtract(url, {}, { signal: AbortSignal.timeout(FETCH_MS) })
-    if (!d) return null
-    return {
-      title: d.title || d.author_name || '',
-      description: '',
-      provider: d.provider_name || '',
-      author: d.author_name || '',
-      type: d.type || 'rich',
-      thumb: d.thumbnail_url || '',
-      source: 'oembed',
-    }
-  } catch {
-    return null // not an oEmbed provider (or fetch failed) → fall back
-  }
-}
-
-const fromOpenGraph = async url => {
-  try {
-    // Walk redirects ourselves so every hop is checked, then hand the library
-    // the final URL with redirects off. It cannot be led anywhere we did not look.
-    const probe = await safeFetch(url, { method: 'GET', signal: AbortSignal.timeout(FETCH_MS) })
-    const finalUrl = probe.url || url
-    const r = await ogs({ url: finalUrl, fetchOptions: { signal: AbortSignal.timeout(FETCH_MS), redirect: 'manual' } })
-    if (!r || r.error || !r.result) return null
-    const x = r.result
-    const img =
-      (Array.isArray(x.ogImage) && x.ogImage[0] && x.ogImage[0].url) ||
-      (Array.isArray(x.twitterImage) && x.twitterImage[0] && x.twitterImage[0].url) ||
-      ''
-    return {
-      title: x.ogTitle || x.twitterTitle || x.dcTitle || '',
-      description: x.ogDescription || x.twitterDescription || '',
-      provider: x.ogSiteName || '',
-      author: x.author || '',
-      type: x.ogType || 'website',
-      thumb: img,
-      favicon: x.favicon || '',
-      source: 'opengraph',
+// ── Outposts ─────────────────────────────────────────────────────────────────
+const cache = new Map() // file -> { mtimeMs, list }
+const readOutposts = statusDir => {
+  const file = statusDir ? path.join(statusDir, 'outpost.json') : null
+  let list = []
+  if (file) {
+    let stat = null
+    try { stat = fs.statSync(file) } catch { cache.delete(file) }
+    if (stat) {
+      const hit = cache.get(file)
+      if (hit && hit.mtimeMs === stat.mtimeMs) list = hit.list
+      else {
+        try {
+          const j = JSON.parse(fs.readFileSync(file, 'utf8'))
+          const v = j && j.unfurl
+          list = Array.isArray(v) ? v : typeof v === 'string' ? [v] : []
+        } catch (e) {
+          console.log(`caution: ${file}: ${e.message}`)
+          list = []
+        }
+        cache.set(file, { mtimeMs: stat.mtimeMs, list })
+      }
     }
-  } catch {
-    return null
   }
+  if (!list.length && process.env.OUTPOST_URL) list = process.env.OUTPOST_URL.split(',')
+  return list.map(s => String(s || '').trim().replace(/\/+$/, '')).filter(Boolean)
 }
 
-// ── route ─────────────────────────────────────────────────────────────────────
+// ── routes ───────────────────────────────────────────────────────────────────
 export const startServer = ({ app, argv }) => {
-  // argv.status = {farm}/{thisDomain}/status → primary farm root is two levels up.
   const primaryRoot = argv && argv.status ? path.dirname(path.dirname(argv.status)) : ''
-  console.log('[wiki-plugin-hitchhiker] registering GET /system/unfurl.json + /system/farm-domains.json')
-
+  const statusDir = argv && argv.status ? argv.status : ''
+  console.log('[wiki-plugin-hitchhiker] registering GET /system/farm-domains.json + /system/outposts.json')
   const cors = res => {
     res.setHeader('Access-Control-Allow-Origin', '*')
     res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS')
   }
-
-  app.options('/system/farm-domains.json', (req, res) => {
-    cors(res)
-    res.sendStatus(204)
-  })
-  app.get('/system/farm-domains.json', (req, res) => {
-    cors(res)
-    res.json(listFarmDomains(primaryRoot))
-  })
-
-  app.options('/system/unfurl.json', (req, res) => {
-    cors(res)
-    res.sendStatus(204)
-  })
-
-  app.get('/system/unfurl.json', async (req, res) => {
-    cors(res)
-    const url = String(req.query.url || '')
-    let host = ''
-    try {
-      host = new URL(url).hostname
-    } catch {}
-
-    const hit = CACHE.get(url)
-    if (hit && Date.now() - hit.t < TTL) return res.json(hit.v)
-
-    let card
-    try {
-      await assertPublicUrl(url)
-      const meta = (await fromOembed(url)) || (await fromOpenGraph(url)) || {}
-      const image = await inlineImage(meta.thumb)
-      card = {
-        url,
-        title: meta.title || host || url,
-        description: meta.description || '',
-        provider: meta.provider || host || '',
-        author: meta.author || '',
-        type: meta.type || 'link',
-        image,
-        favicon: meta.favicon || '',
-        source: meta.source || 'fallback',
-      }
-    } catch (e) {
-      // Graceful degradation (bad/blocked/offline URL) — always return a card.
-      card = {
-        url,
-        title: host || url,
-        description: '',
-        provider: host,
-        author: '',
-        type: 'link',
-        image: '',
-        favicon: '',
-        source: 'error',
-        error: String((e && e.message) || e),
-      }
-    }
-    CACHE.set(url, { t: Date.now(), v: card })
-    res.json(card)
-  })
+  app.options('/system/farm-domains.json', (req, res) => { cors(res); res.sendStatus(204) })
+  app.get('/system/farm-domains.json', (req, res) => { cors(res); res.json(listFarmDomains(primaryRoot)) })
+  app.options('/system/outposts.json', (req, res) => { cors(res); res.sendStatus(204) })
+  app.get('/system/outposts.json', (req, res) => { cors(res); res.json({ unfurl: readOutposts(statusDir) }) })
 }