tiptapify 0.3.5 → 0.4.1

package/src/extensions/components/ai/backend.test.ts

@@ -176,6 +176,7 @@ describe('createAiBackendProvider', () => {
     const provider = createAiBackendProvider({ endpoint: '/api' })
 
     await expect(provider(createRequest(), createContext())).rejects.toThrow('Model is busy')
+    await expect(provider(createRequest(), createContext())).rejects.toMatchObject({ status: 500 })
   })
 
   it('throws a status error when the error body is not JSON', async () => {

package/src/extensions/components/ai/backend.ts

@@ -144,7 +144,8 @@ async function createBackendError(response: Response): Promise<Error> {
     // Keep the raw body
   }
 
-  return new Error(message || `Request failed with status ${response.status}`)
+  // The status lets the dialog explain the error (see errors.ts)
+  return Object.assign(new Error(message || `Request failed with status ${response.status}`), { status: response.status })
 }
 
 function parseBackendResponse(text: string): TiptapifyAiResponse | TiptapifyAiOpenAiResponse | string {

package/src/extensions/components/ai/context.test.ts

@@ -0,0 +1,149 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import { Editor } from '@tiptap/core'
+import { Document } from '@tiptap/extension-document'
+import { Paragraph } from '@tiptap/extension-paragraph'
+import { Text } from '@tiptap/extension-text'
+import { countWords, describeAiContext, excerpt, selectedTextOf, textAround, wholeWordRange } from './context'
+
+let editor: Editor | null = null
+
+function docOf(content: string) {
+  editor = new Editor({ content, extensions: [Document, Paragraph, Text] })
+
+  return editor.state.doc
+}
+
+afterEach(() => {
+  editor?.destroy()
+  editor = null
+})
+
+describe('excerpt', () => {
+  it('keeps short text whole on one line', () => {
+    expect(excerpt('  Hello\n  world  ')).toBe('Hello world')
+  })
+
+  it('cuts after the first words', () => {
+    expect(excerpt('one two three four five six seven eight')).toBe('one two three four five six…')
+  })
+
+  it('cuts a very long word run by characters', () => {
+    expect(excerpt('a'.repeat(100), 6, 10)).toBe(`${'a'.repeat(10)}…`)
+  })
+})
+
+describe('countWords', () => {
+  it('counts words separated by any whitespace', () => {
+    expect(countWords('')).toBe(0)
+    expect(countWords('  one\ttwo\n\nthree ')).toBe(3)
+  })
+})
+
+describe('describeAiContext', () => {
+  it('describes a selection over two paragraphs', () => {
+    const doc = docOf('<p>Hello world</p><p>Second part</p>')
+    // From inside the first paragraph to the middle of the second
+    const info = describeAiContext(doc, 7, 20)
+
+    expect(info).toEqual({
+      kind: 'selection',
+      paragraphs: 2,
+      characters: 'world'.length + 'Second'.length,
+      excerpt: 'world Second',
+    })
+  })
+
+  it('does not count a paragraph the selection only touches at its start', () => {
+    const doc = docOf('<p>Hello</p><p>World</p>')
+    // From the start of "Hello" to the start of "World"
+    expect(describeAiContext(doc, 1, 8).paragraphs).toBe(1)
+  })
+
+  it('falls back to the whole text without a selection, as the request does', () => {
+    const doc = docOf('<p>Hello</p>')
+
+    expect(describeAiContext(doc, 3, 3).kind).toBe('document')
+    expect(selectedTextOf(doc, 3, 3)).toBe('')
+  })
+
+  it('is only the cursor in an empty document', () => {
+    expect(describeAiContext(docOf('<p></p>'), 1, 1).kind).toBe('cursor')
+  })
+
+  it('sends the selected blocks separated by new lines', () => {
+    const doc = docOf('<p>Hello</p><p>World</p>')
+
+    expect(selectedTextOf(doc, 0, doc.content.size)).toBe('Hello\nWorld')
+  })
+})
+
+describe('wholeWordRange', () => {
+  // Positions of `<p>…</p>`: the text starts at 1
+  const text = 'The talks has entered a new stage.'
+  const at = (part: string) => 1 + text.indexOf(part)
+
+  function selected(from: number, to: number) {
+    const doc = docOf(`<p>${text}</p>`)
+    const range = wholeWordRange(doc, from, to)
+
+    return doc.textBetween(range.from, range.to)
+  }
+
+  it('takes the whole word a selection ends inside of', () => {
+    expect(selected(1, at('ew stage'))).toBe('The talks has entered a new')
+  })
+
+  it('takes the whole word a selection starts inside of', () => {
+    expect(selected(at('alks'), at(' has'))).toBe('talks')
+  })
+
+  it('keeps a selection on word boundaries', () => {
+    expect(selected(at('talks'), at(' has'))).toBe('talks')
+    expect(selected(at(' has'), at(' entered'))).toBe(' has')
+    expect(selected(at('stage'), at('.') + 1)).toBe('stage.')
+  })
+
+  it('widens a selection inside one word to the word', () => {
+    expect(selected(at('nter'), at('red'))).toBe('entered')
+  })
+
+  it('keeps an apostrophe inside a word', () => {
+    const doc = docOf('<p>We don\'t stop</p>')
+    const range = wholeWordRange(doc, 1, 6)
+
+    expect(doc.textBetween(range.from, range.to)).toBe('We don\'t')
+  })
+
+  it('leaves a collapsed selection and one across paragraphs on boundaries alone', () => {
+    const doc = docOf('<p>one two</p><p>three four</p>')
+
+    expect(wholeWordRange(doc, 3, 3)).toEqual({ from: 3, to: 3 })
+    expect(doc.textBetween(...Object.values(wholeWordRange(doc, 6, 13)) as [number, number], '\n')).toBe('two\nthree')
+  })
+})
+
+describe('textAround', () => {
+  it('gives the text of the selection\'s paragraph before and after it', () => {
+    const doc = docOf('<p>First one.</p><p>The talks has entered a new stage.</p><p>Last one.</p>')
+    const from = 13 + 'The talks '.length
+    const to = from + 'has'.length
+
+    expect(doc.textBetween(from, to)).toBe('has')
+    expect(textAround(doc, from, to)).toEqual({ before: 'The talks ', after: ' entered a new stage.' })
+  })
+
+  it('gives nothing around a whole paragraph or a collapsed selection', () => {
+    const doc = docOf('<p>one two</p>')
+
+    expect(textAround(doc, 1, 8)).toEqual({ before: '', after: '' })
+    expect(textAround(doc, 3, 3)).toEqual({ before: '', after: '' })
+  })
+
+  it('takes at most 500 characters on each side, the nearest ones', () => {
+    const doc = docOf(`<p>${'a'.repeat(600)}X${'b'.repeat(600)}</p>`)
+    const { before, after } = textAround(doc, 601, 602)
+
+    expect(before).toBe('a'.repeat(500))
+    expect(after).toBe('b'.repeat(500))
+  })
+})

package/src/extensions/components/ai/context.ts

@@ -0,0 +1,145 @@
+import type { Node } from '@tiptap/pm/model'
+
+/**
+ * What a request works on: the selected text, or (no selection) the whole document with the result
+ * going to the cursor; an empty document gives only the cursor.
+ */
+export type AiContextKind = 'selection' | 'document' | 'cursor'
+
+export type AiContextInfo = {
+  kind: AiContextKind,
+  paragraphs: number,
+  characters: number,
+  excerpt: string,
+}
+
+const EXCERPT_WORDS = 6
+const EXCERPT_CHARS = 60
+
+export function words(text: string): string[] {
+  return text.split(/\s+/).filter(Boolean)
+}
+
+export function countWords(text: string): number {
+  return words(text).length
+}
+
+/** The first words of `text` on one line, with an ellipsis when something is cut off */
+export function excerpt(text: string, maxWords = EXCERPT_WORDS, maxChars = EXCERPT_CHARS): string {
+  const all = words(text)
+  let value = all.slice(0, maxWords).join(' ')
+  let cut = all.length > maxWords
+
+  if (value.length > maxChars) {
+    value = value.slice(0, maxChars).trimEnd()
+    cut = true
+  }
+
+  return cut ? `${value}…` : value
+}
+
+// A letter, a mark or a digit, and an apostrophe inside a word (don't, l'homme)
+const WORD_CHAR = /[\p{L}\p{M}\p{N}_'’]/u
+
+// One character per position of the text block's content: text as is, an inline leaf (image, break) as one char
+const LEAF_CHAR = '\uFFFC'
+
+function isWordAt(text: string, index: number): boolean {
+  return index >= 0 && index < text.length && WORD_CHAR.test(text[index])
+}
+
+/**
+ * The selection widened to whole words: a selection that starts or ends inside a word takes the whole word,
+ * or the result replacing it would leave the rest of the word behind.
+ * A collapsed selection, or one already on word boundaries, stays as is.
+ */
+export function wholeWordRange(doc: Node, from: number, to: number): { from: number, to: number } {
+  if (from === to) {
+    return { from, to }
+  }
+
+  const widen = (pos: number, side: -1 | 1): number => {
+    const $pos = doc.resolve(pos)
+    if (!$pos.parent.isTextblock) {
+      return pos
+    }
+
+    const text = $pos.parent.textBetween(0, $pos.parent.content.size, undefined, LEAF_CHAR)
+    let offset = $pos.parentOffset
+    if (!isWordAt(text, offset - 1) || !isWordAt(text, offset)) {
+      return pos
+    }
+
+    while (side < 0 ? isWordAt(text, offset - 1) : isWordAt(text, offset)) {
+      offset += side
+    }
+
+    return $pos.start() + offset
+  }
+
+  return { from: widen(from, -1), to: widen(to, 1) }
+}
+
+const AROUND_CHARS = 500
+
+/**
+ * The text of the selection's paragraphs before and after it, at most `AROUND_CHARS` on each side (cut at the
+ * far end).
+ */
+export function textAround(doc: Node, from: number, to: number): { before: string, after: string } {
+  if (from === to) {
+    return { before: '', after: '' }
+  }
+
+  const $from = doc.resolve(from)
+  const $to = doc.resolve(to)
+  const before = $from.parent.isTextblock ? doc.textBetween($from.start(), from, '\n') : ''
+  const after = $to.parent.isTextblock ? doc.textBetween(to, $to.end(), '\n') : ''
+
+  return { before: before.slice(-AROUND_CHARS), after: after.slice(0, AROUND_CHARS) }
+}
+
+/**
+ * The same text the dialog sends as `selectedText` (block boundaries as new lines):
+ * an empty string means the request falls back to the whole document
+ */
+export function selectedTextOf(doc: Node, from: number, to: number): string {
+  return from === to ? '' : doc.textBetween(from, to, '\n')
+}
+
+export function describeAiContext(doc: Node, from: number, to: number): AiContextInfo {
+  const selectedText = selectedTextOf(doc, from, to)
+
+  if (!selectedText) {
+    return {
+      kind: doc.textContent.trim() ? 'document' : 'cursor',
+      paragraphs: 0,
+      characters: 0,
+      excerpt: '',
+    }
+  }
+
+  // A text block counts only when some of its text is selected: a selection that
+  // ends at the very start of the next paragraph does not count that paragraph
+  let paragraphs = 0
+  doc.nodesBetween(from, to, (node, pos) => {
+    if (!node.isTextblock) {
+      return true
+    }
+
+    const start = Math.max(from, pos + 1)
+    const end = Math.min(to, pos + 1 + node.content.size)
+    if (end > start) {
+      paragraphs++
+    }
+
+    return false
+  })
+
+  return {
+    kind: 'selection',
+    paragraphs: Math.max(paragraphs, 1),
+    characters: doc.textBetween(from, to, '').length,
+    excerpt: excerpt(selectedText),
+  }
+}

package/src/extensions/components/ai/diff.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it } from 'vitest'
+import { changePercent, diffWords, formatChangePercent } from './diff'
+
+function render(original: string, revised: string) {
+  return diffWords(original, revised)
+    .map(({ type, text }) => (type === 'equal' ? text : type === 'insert' ? `[+${text}]` : `[-${text}]`))
+    .join('')
+}
+
+describe('diffWords', () => {
+  it('marks a replaced word as a deletion followed by an insertion', () => {
+    expect(render('the quick fox', 'the slow fox')).toBe('the [-quick ][+slow ]fox')
+  })
+
+  it('marks added and removed words', () => {
+    expect(render('a b c', 'a c d')).toBe('a [-b ]c [+d]')
+  })
+
+  it('keeps the revised text when only the equal parts are joined with insertions', () => {
+    const revised = 'Completely new wording here'
+    const segments = diffWords('Some old text', revised)
+
+    expect(segments.filter(s => s.type !== 'delete').map(s => s.text).join('')).toBe(revised)
+    expect(segments.filter(s => s.type !== 'insert').map(s => s.text.trim()).join(' ')).toBe('Some old text')
+  })
+
+  it('keeps a space before a deleted last word', () => {
+    expect(render('fox jumps', 'fox')).toBe('fox [-jumps]')
+  })
+
+  it('compares words regardless of the whitespace around them', () => {
+    expect(diffWords('one\ntwo', ' one two ')).toEqual([{ type: 'equal', text: 'one two' }])
+  })
+
+  it('handles empty sides', () => {
+    expect(diffWords('', 'new text')).toEqual([{ type: 'insert', text: 'new text' }])
+    expect(diffWords('old', '')).toEqual([{ type: 'delete', text: 'old' }])
+    expect(diffWords('', '')).toEqual([])
+  })
+
+  it('falls back to a whole delete and insert for a very large change', () => {
+    const original = Array.from({ length: 1200 }, (_, i) => `a${i}`).join(' ')
+    const revised = Array.from({ length: 1200 }, (_, i) => `b${i}`).join(' ')
+
+    expect(diffWords(original, revised).map(s => s.type)).toEqual(['delete', 'insert'])
+  })
+})
+
+describe('changePercent', () => {
+  it('is the length change against the original in whole percent', () => {
+    expect(changePercent('1234567890', '12345')).toBe(-50)
+    expect(changePercent('abc', 'abcd')).toBe(33)
+    expect(changePercent(' abc ', 'abc')).toBe(0)
+  })
+
+  it('is null without an original', () => {
+    expect(changePercent('  ', 'text')).toBeNull()
+  })
+
+  it('formats with a sign', () => {
+    expect(formatChangePercent(12)).toBe('+12%')
+    expect(formatChangePercent(-8)).toBe('−8%')
+    expect(formatChangePercent(0)).toBe('0%')
+  })
+})

package/src/extensions/components/ai/diff.ts

@@ -0,0 +1,120 @@
+export type DiffSegment = {
+  type: 'equal' | 'insert' | 'delete',
+  text: string,
+}
+
+// Above this many LCS cells (≈ 1000 × 1000 changed words) the middle part is shown
+// as a plain delete + insert instead of a word-level diff, to keep the dialog responsive
+const MAX_CELLS = 1_000_000
+
+function tokenize(text: string): string[] {
+  return text.trim().match(/\S+\s*/g) ?? []
+}
+
+function word(token: string): string {
+  return token.trimEnd()
+}
+
+/**
+ * A word diff of `revised` against `original`: words with their trailing whitespace,
+ * the longest common subsequence after the common prefix and suffix are cut off.
+ * Adjacent segments of one type are merged; deletions come before insertions.
+ */
+export function diffWords(original: string, revised: string): DiffSegment[] {
+  const a = tokenize(original)
+  const b = tokenize(revised)
+  const segments: DiffSegment[] = []
+
+  function push(type: DiffSegment['type'], text: string) {
+    const last = segments[segments.length - 1]
+    if (last?.type === type) {
+      last.text += text
+    } else {
+      segments.push({ type, text })
+    }
+  }
+
+  // An equal word keeps the whitespace after it from either side, so a deletion
+  // right after the last word of the revision does not stick to it
+  function pushEqual(i: number, j: number) {
+    push('equal', /\s$/.test(b[j]) ? b[j] : a[i])
+  }
+
+  let start = 0
+  while (start < a.length && start < b.length && word(a[start]) === word(b[start])) {
+    start++
+  }
+
+  let endA = a.length
+  let endB = b.length
+  while (endA > start && endB > start && word(a[endA - 1]) === word(b[endB - 1])) {
+    endA--
+    endB--
+  }
+
+  for (let k = 0; k < start; k++) {
+    pushEqual(k, k)
+  }
+
+  const n = endA - start
+  const m = endB - start
+
+  if (n * m > MAX_CELLS) {
+    a.slice(start, endA).forEach(token => push('delete', token))
+    b.slice(start, endB).forEach(token => push('insert', token))
+  } else {
+    // lengths[i * (m + 1) + j]: the LCS length of a[start + i..endA) and b[start + j..endB)
+    const lengths = new Uint32Array((n + 1) * (m + 1))
+    for (let i = n - 1; i >= 0; i--) {
+      for (let j = m - 1; j >= 0; j--) {
+        lengths[i * (m + 1) + j] = word(a[start + i]) === word(b[start + j])
+          ? lengths[(i + 1) * (m + 1) + j + 1] + 1
+          : Math.max(lengths[(i + 1) * (m + 1) + j], lengths[i * (m + 1) + j + 1])
+      }
+    }
+
+    let i = 0
+    let j = 0
+    while (i < n && j < m) {
+      if (word(a[start + i]) === word(b[start + j])) {
+        pushEqual(start + i, start + j)
+        i++
+        j++
+      } else if (lengths[(i + 1) * (m + 1) + j] >= lengths[i * (m + 1) + j + 1]) {
+        push('delete', a[start + i++])
+      } else {
+        push('insert', b[start + j++])
+      }
+    }
+    while (i < n) {
+      push('delete', a[start + i++])
+    }
+    while (j < m) {
+      push('insert', b[start + j++])
+    }
+  }
+
+  for (let k = 0; k < a.length - endA; k++) {
+    pushEqual(endA + k, endB + k)
+  }
+
+  return segments
+}
+
+/** How much longer (positive) or shorter (negative) `revised` is, in whole percent; null without an original */
+export function changePercent(original: string, revised: string): number | null {
+  const before = original.trim().length
+  if (!before) {
+    return null
+  }
+
+  return Math.round((revised.trim().length - before) / before * 100)
+}
+
+export function formatChangePercent(percent: number): string {
+  if (percent > 0) {
+    return `+${percent}%`
+  }
+
+  return percent < 0 ? `−${-percent}%` : '0%'
+}

package/src/extensions/components/ai/errors.test.ts

@@ -0,0 +1,45 @@
+import { describe, expect, it } from 'vitest'
+import { aiErrorKind, aiErrorDetail } from '@tiptapify/extensions/components/ai/errors'
+
+const withStatus = (status: number, message = '') => Object.assign(new Error(message), { status })
+
+describe('AI errors', () => {
+  it.each([
+    ['Chrome', new TypeError('Failed to fetch')],
+    ['Firefox', new TypeError('NetworkError when attempting to fetch resource.')],
+    ['Safari', new TypeError('Load failed')],
+    ['Node', new TypeError('fetch failed')],
+  ])('reads a failed fetch (%s) as the service being unreachable or blocked by CORS', (_, error) => {
+    expect(aiErrorKind(error)).toBe('network')
+    expect(aiErrorDetail(error)).toBe('')
+  })
+
+  it.each([
+    [401, 'auth'],
+    [403, 'auth'],
+    [404, 'not_found'],
+    [408, 'timeout'],
+    [429, 'rate_limit'],
+    [500, 'unavailable'],
+    [503, 'unavailable'],
+    [504, 'timeout'],
+    [400, 'request'],
+  ])('reads status %i as %s', (status, kind) => {
+    expect(aiErrorKind(withStatus(status as number))).toBe(kind)
+  })
+
+  it('reads a timeout of the request', () => {
+    expect(aiErrorKind(new DOMException('The operation timed out.', 'TimeoutError'))).toBe('timeout')
+  })
+
+  it('keeps the message of the service as the detail, not the generic status text', () => {
+    expect(aiErrorDetail(withStatus(401, 'missing or wrong API key'))).toBe('missing or wrong API key')
+    expect(aiErrorDetail(withStatus(503, 'Request failed with status 503'))).toBe('')
+  })
+
+  it('shows any other error of a custom provider as it is', () => {
+    expect(aiErrorKind(new Error('Quota of the project is used up'))).toBe('other')
+    expect(aiErrorDetail(new Error('Quota of the project is used up'))).toBe('Quota of the project is used up')
+    expect(aiErrorKind('weird')).toBe('other')
+  })
+})

package/src/extensions/components/ai/errors.ts

@@ -0,0 +1,67 @@
+// What went wrong with an AI request, to explain it in the dialog (`ai.errors.<kind>`)
+export type AiErrorKind = 'network' | 'auth' | 'not_found' | 'timeout' | 'rate_limit' | 'unavailable' | 'request' | 'other'
+
+// The messages browsers give a fetch that never got a response: the service is down, unreachable, or CORS
+// blocked the answer (the browser hides which; its console tells)
+const FAILED_FETCH = /^(failed to fetch|networkerror when attempting to fetch resource\.?|load failed|fetch failed)$/i
+
+// The text backend.ts gives a failed response without a message of its own
+const STATUS_ONLY = /^Request failed with status \d+$/
+
+function statusOf(error: unknown): number | null {
+  const status = (error as { status?: unknown } | null)?.status
+
+  return typeof status === 'number' ? status : null
+}
+
+/**
+ * The kind of a failed AI request: a failed fetch, the HTTP status of the response (backend.ts sets
+ * `status` on its errors; a custom `aiProvider` may throw an error with `status` too), or a timeout.
+ */
+export function aiErrorKind(error: unknown): AiErrorKind {
+  // A DOMException (AbortSignal.timeout) is not an Error everywhere
+  if ((error as { name?: unknown } | null)?.name === 'TimeoutError') {
+    return 'timeout'
+  }
+
+  if (error instanceof TypeError && FAILED_FETCH.test(error.message.trim())) {
+    return 'network'
+  }
+
+  const status = statusOf(error)
+  if (status === null) {
+    return 'other'
+  }
+
+  if (status === 401 || status === 403) {
+    return 'auth'
+  }
+
+  if (status === 404) {
+    return 'not_found'
+  }
+
+  if (status === 408 || status === 504) {
+    return 'timeout'
+  }
+
+  if (status === 429) {
+    return 'rate_limit'
+  }
+
+  return status >= 500 ? 'unavailable' : 'request'
+}
+
+/**
+ * The message of the service (or of a custom provider) shown after the explanation; empty when it only
+ * repeats the status or the browser's network message.
+ */
+export function aiErrorDetail(error: unknown): string {
+  if (!(error instanceof Error) || aiErrorKind(error) === 'network') {
+    return ''
+  }
+
+  const message = error.message.trim()
+
+  return STATUS_ONLY.test(message) ? '' : message
+}