switchroom 0.16.38 → 0.16.46

package/telegram-plugin/tests/paragraph-normalizer.test.ts

@@ -48,11 +48,71 @@ describe('normalizeParagraphBreaks', () => {
     expect(normalizeParagraphBreaks(input)).toBe(input)
   })
 
-  test('does NOT promote when the next line is a list item (tight list stays tight)', () => {
-    const input = 'Here are the steps.\n- first\n- second\n- third'
+  test('promotes every interior break in a vertical stat card (#2750)', () => {
+    // The gymbro food-log card: lines end in digits/letters (no terminator),
+    // but each is a standalone `label: value` stat line, so every interior
+    // break must become a GFM hard break instead of collapsing to one wall.
+    const input = 'Calories: 1800\nProtein: 120g\nCarbs: 200g'
+    expect(normalizeParagraphBreaks(input)).toBe(
+      'Calories: 1800  \nProtein: 120g  \nCarbs: 200g',
+    )
+  })
+
+  test('promotes a key-value line followed by prose (#2750)', () => {
+    const input = 'Status: active\nEverything is running smoothly.'
+    expect(normalizeParagraphBreaks(input)).toBe(
+      'Status: active  \nEverything is running smoothly.',
+    )
+  })
+
+  test('promotes bold-label stat cards — colon inside AND outside the emphasis (#2750)', () => {
+    // The fleet HOUSE style uses markdown-bold labels. Both `**Calories:** 1800`
+    // (colon inside the bold) and `**Calories**: 1800` (colon outside) must have
+    // every interior break promoted.
+    const inside = '**Calories:** 1800\n**Protein:** 120g\n**Carbs:** 200g'
+    expect(normalizeParagraphBreaks(inside)).toBe(
+      '**Calories:** 1800  \n**Protein:** 120g  \n**Carbs:** 200g',
+    )
+    const outside = '**Calories**: 1800\n**Protein**: 120g'
+    expect(normalizeParagraphBreaks(outside)).toBe('**Calories**: 1800  \n**Protein**: 120g')
+  })
+
+  test('regression guard: a mid-sentence colon soft-wrap still does NOT promote (#2750)', () => {
+    // A wrapped sentence that happens to contain a colon must NOT be mistaken
+    // for a stat line: the next line continues the sentence in lowercase.
+    const input = 'He made one point: the plan was sound and\nthe timing was right too'
+    expect(normalizeParagraphBreaks(input)).toBe(input)
+  })
+
+  test('regression guard: a long colon-clause with a CAPITAL continuation does NOT promote (#2750)', () => {
+    // The lowercase-continuation guard alone would miss this: the wrapped clause
+    // continues with a capitalised proper noun. The label before the colon is a
+    // long multi-word run (not a short stat label), so the word/length cap keeps
+    // it from being mistaken for a stat line.
+    const input = 'The deal has one catch: the buyer wants it and\nToronto lawyers must sign off'
     expect(normalizeParagraphBreaks(input)).toBe(input)
   })
 
+  test('lowercase-label stat cards are an ACCEPTED false-negative (not promoted) (#2750)', () => {
+    // `calories: 1800\nprotein: 120g` — the lowercase-started next line is
+    // indistinguishable from a mid-sentence soft-wrap continuation, so the
+    // conservative guard leaves it alone. Documented, not fixed: models are
+    // steered toward capitalised / bold labels, and a false-negative (un-promoted
+    // break) is the module's preferred failure mode over a false-positive.
+    const input = 'calories: 1800\nprotein: 120g'
+    expect(normalizeParagraphBreaks(input)).toBe(input)
+  })
+
+  test('list glued to prose gains a block-boundary blank line; interior stays tight', () => {
+    // Uniform-block-spacing: a list start glued to prose by a single `\n`
+    // gets a blank line (so the spacer pass can see the transition); the
+    // list's own single-`\n` interior is never touched.
+    const input = 'Here are the steps.\n- first\n- second\n- third'
+    expect(normalizeParagraphBreaks(input)).toBe(
+      'Here are the steps.\n\n- first\n- second\n- third',
+    )
+  })
+
   test('does NOT promote between list items (ordered or unordered)', () => {
     const ul = '- alpha.\n- beta.\n- gamma.'
     const ol = '1. first.\n2. second.\n3. third.'
@@ -275,6 +335,37 @@ describe('normalizeParagraphBreaks', () => {
     const input = '- first item\n    continued text of the first item'
     expect(normalizeParagraphBreaks(input)).toBe(input)
   })
+
+  test('F2: prose with an interior ` | ` gets normal paragraph-break treatment (not table)', () => {
+    // `isMarkerLine` used to treat any line containing ` | ` as a table row and
+    // suppress the paragraph-break promotion. A sentence like "choose A | B" is
+    // prose, not a table — the lone `\n` after a terminated prose line must
+    // still be promoted to a GFM hard break.
+    const input = 'Pick one option, choose A | B here.\nThen continue below.'
+    expect(normalizeParagraphBreaks(input)).toBe(
+      'Pick one option, choose A | B here.  \nThen continue below.',
+    )
+  })
+
+  test('F2: a genuine table (header + delimiter) is still detected as structural', () => {
+    // The tightened check must still recognise a real GFM table: a blank line
+    // is inserted before the header and the rows stay contiguous.
+    const input = 'Here is data.\n| a | b |\n| --- | --- |\n| 1 | 2 |'
+    expect(normalizeParagraphBreaks(input)).toBe(
+      'Here is data.\n\n| a | b |\n| --- | --- |\n| 1 | 2 |',
+    )
+  })
+
+  test('F4: blank line inserted after a closed code fence before glued prose', () => {
+    // Prose glued directly onto a fence close (single `\n`) can be swallowed /
+    // mis-parsed. A blank line after a closed fence is always CommonMark-safe.
+    const input = 'Intro.\n\n```\nconst x = 1\n```\nThen the prose continues.'
+    const out = normalizeParagraphBreaks(input)
+    // The fence interior is untouched...
+    expect(out).toContain('```\nconst x = 1\n```')
+    // ...and a blank line now separates the fence close from the following prose.
+    expect(out).toContain('```\n\nThen the prose continues.')
+  })
 })
 
 // ---------------------------------------------------------------------------
@@ -383,40 +474,48 @@ describe('addParagraphSpacers', () => {
     expect(addParagraphSpacers(input)).toBe(input)
   })
 
-  test('does NOT space a gap adjacent to a list (tight list rhythm preserved)', () => {
-    const input = 'Here are the steps.\n\n- first\n- second'
-    expect(addParagraphSpacers(input)).toBe(input)
-    const input2 = '- first\n- second\n\nClosing prose after the list.'
-    expect(addParagraphSpacers(input2)).toBe(input2)
+  test('spaces prose↔list transitions (uniform block spacing), list interior tight', () => {
+    expect(addParagraphSpacers('Here are the steps.\n\n- first\n- second')).toBe(
+      `Here are the steps.${gap}- first\n- second`,
+    )
+    expect(addParagraphSpacers('- first\n- second\n\nClosing prose after the list.')).toBe(
+      `- first\n- second${gap}Closing prose after the list.`,
+    )
   })
 
-  test('does NOT space a gap adjacent to a table', () => {
-    const input = 'Intro.\n\n| a | b |\n| --- | --- |\n| 1 | 2 |'
-    expect(addParagraphSpacers(input)).toBe(input)
+  test('spaces a prose→table boundary; table rows stay contiguous', () => {
+    const table = '| a | b |\n| --- | --- |\n| 1 | 2 |'
+    expect(addParagraphSpacers(`Intro.\n\n${table}`)).toBe(`Intro.${gap}${table}`)
   })
 
-  test('does NOT space a gap adjacent to a fenced code block', () => {
+  test('spaces prose↔fence boundaries; fence interior untouched', () => {
     const input = 'Look here.\n\n```js\nconst a = 1;\n```\n\nDone.'
-    // Neither the prose→fence nor the fence→prose gap gets a spacer.
-    expect(addParagraphSpacers(input)).toBe(input)
+    expect(addParagraphSpacers(input)).toBe(
+      `Look here.${gap}\`\`\`js\nconst a = 1;\n\`\`\`${gap}Done.`,
+    )
   })
 
-  test('does NOT space a gap adjacent to a heading or blockquote', () => {
+  test('spaces heading→anything and blockquote boundaries', () => {
     expect(addParagraphSpacers('Intro prose.\n\n## Section\n\nBody prose.')).toBe(
-      'Intro prose.\n\n## Section\n\nBody prose.',
+      `Intro prose.${gap}## Section${gap}Body prose.`,
     )
     expect(addParagraphSpacers('Said.\n\n> a quote\n\nAfter.')).toBe(
-      'Said.\n\n> a quote\n\nAfter.',
+      `Said.${gap}> a quote${gap}After.`,
     )
   })
 
+  test('does NOT space between items of the same loose list / same-kind blocks', () => {
+    const looseList = '- first\n\n- second\n\n- third'
+    expect(addParagraphSpacers(looseList)).toBe(looseList)
+    const quotes = '> one\n\n> two'
+    expect(addParagraphSpacers(quotes)).toBe(quotes)
+  })
+
   test('never reaches inside a fenced block (interior blank line untouched)', () => {
     const input = 'Before.\n\n```\nline 1\n\nline 2\n```\n\nAfter.'
     const out = addParagraphSpacers(input)
     // The fence interior — including its own blank line — is byte-for-byte intact.
     expect(out).toContain('```\nline 1\n\nline 2\n```')
-    // And no spacer paragraph was injected anywhere (both gaps touch the fence).
-    expect(out).not.toContain(PARAGRAPH_SPACER)
   })
 
   test('a body with no paragraph gap is returned unchanged', () => {
@@ -435,9 +534,9 @@ describe('addParagraphSpacers', () => {
     const out = addParagraphSpacers(normalizeParagraphBreaks(input))
     // The two prose paragraphs gain a visible spacer between them.
     expect(out).toContain(`Summary of the change.${gap}It does two things.`)
-    // The list stays tight (no spacer wedged before it).
-    expect(out).toContain('- adds a normalizer\n- lifts the cap')
-    expect(out).not.toContain(`${PARAGRAPH_SPACER}\n\n- adds`)
+    // The prose→list transition gains a spacer too (uniform block spacing),
+    // but the list INTERIOR stays tight.
+    expect(out).toContain(`It does two things.${gap}- adds a normalizer\n- lifts the cap`)
   })
 })
 
@@ -470,3 +569,86 @@ describe('normalizeParagraphBreaks — inline bullet split integration', () => {
     expect(twice).toBe(once)
   })
 })
+
+/**
+ * Regression guard for the stray whitespace-only blank line between paragraphs
+ * (the "\n\n \n\n" seen in a real reply). A model (or an upstream transform)
+ * that authors a blank line whose only content is ASCII whitespace leaves a
+ * ragged / oversized gap: CommonMark discards it so it buys no visible space,
+ * but it reads as noise in the raw text. Step 1 of normalizeParagraphBreaks now
+ * collapses any run of blank lines — including whitespace-only interior lines —
+ * to exactly one clean `\n\n`, WITHOUT ever eating the deliberate U+00A0
+ * paragraph spacer that addParagraphSpacers (#2692) adds later for a visible
+ * gap on the rich-message path.
+ */
+describe('normalizeParagraphBreaks — whitespace-only blank-line collapse (Bug 2)', () => {
+  const NBSP = ' '
+
+  test('a lone-space blank line between paragraphs collapses to a clean `\\n\\n` (real string)', () => {
+    const input =
+      "Here's what was fixed and how it validates.\n\n \n\n" +
+      'One loose end: a stray thing.'
+    expect(normalizeParagraphBreaks(input)).toBe(
+      "Here's what was fixed and how it validates.\n\n" +
+        'One loose end: a stray thing.',
+    )
+  })
+
+  test('a single whitespace-only blank line (`A\\n \\nB`) collapses to `\\n\\n`', () => {
+    expect(normalizeParagraphBreaks('Alpha para.\n \nBravo para.')).toBe(
+      'Alpha para.\n\nBravo para.',
+    )
+    expect(normalizeParagraphBreaks('Alpha para.\n\t\nBravo para.')).toBe(
+      'Alpha para.\n\nBravo para.',
+    )
+  })
+
+  test('multiple stray whitespace-only lines in one gap collapse to a single `\\n\\n`', () => {
+    expect(normalizeParagraphBreaks('A.\n\n \n\n \n\nB.')).toBe('A.\n\nB.')
+  })
+
+  test('a legitimate single `\\n\\n` paragraph break is preserved exactly', () => {
+    expect(normalizeParagraphBreaks('Alpha para.\n\nBravo para.')).toBe(
+      'Alpha para.\n\nBravo para.',
+    )
+  })
+
+  test('never EMITS a whitespace-only line between two prose paragraphs', () => {
+    const out = normalizeParagraphBreaks('First.\n\n  \n\nSecond.')
+    for (const line of out.split('\n')) {
+      // Every line is either empty or has real (non-ASCII-whitespace) content.
+      expect(line === '' || /\S/.test(line) || line.includes(NBSP)).toBe(true)
+    }
+    expect(out).toBe('First.\n\nSecond.')
+  })
+
+  test('a blank line inside a fenced code block (masked) is NOT collapsed', () => {
+    const input = '```\ncode line 1\n \ncode line 2\n```'
+    // The whitespace-only line lives inside a masked fence and must survive.
+    expect(normalizeParagraphBreaks(input)).toBe(input)
+  })
+
+  test('trailing hard-break spaces on a CONTENT line are not eaten', () => {
+    // "line.  \n\nNext." — the two trailing spaces are a GFM hard break on a
+    // content line, not a blank line; they must survive.
+    const input = 'Trailing spaces on line.  \n\nNext.'
+    expect(normalizeParagraphBreaks(input)).toBe(input)
+  })
+
+  test('the deliberate U+00A0 spacer from addParagraphSpacers is preserved (no #2692 regression)', () => {
+    const normalized = normalizeParagraphBreaks('Alpha para.\n\nBravo para.')
+    const spaced = addParagraphSpacers(normalized)
+    // addParagraphSpacers wedges a U+00A0-only line to force a visible gap.
+    expect(spaced).toContain('\n\n' + PARAGRAPH_SPACER + '\n\n')
+    expect(spaced.split('\n')).toContain(NBSP)
+    // And re-running the normalizer must NOT eat that intentional spacer.
+    expect(normalizeParagraphBreaks(spaced)).toBe(spaced)
+  })
+
+  test('idempotent: collapsing a stray gap twice is stable', () => {
+    const once = normalizeParagraphBreaks('A.\n\n \n\nB.')
+    const twice = normalizeParagraphBreaks(once)
+    expect(twice).toBe(once)
+    expect(once).toBe('A.\n\nB.')
+  })
+})