speechready 0.0.1 → 0.1.0

package/README.md

@@ -1,5 +1,92 @@
-# speechready
+# speechready (JavaScript / TypeScript)
 
-Prepare text for text-to-speech: clean LLM output, verbalize ambiguous tokens, compile pauses and emphasis to each provider's syntax, and chunk under model limits.
+Prepare text for text-to-speech. See the [repository README](https://github.com/berkerbatur/speechready) for the project overview and the evaluation results.
 
-This is a placeholder release that reserves the name. The first working version follows at https://github.com/berkerbatur/speechready.
+```
+npm install speechready
+```
+
+ESM only, Node 20 or newer, no runtime dependencies. Type declarations are included. The provider profiles from `spec/profiles` are compiled into the package, so it also runs in a browser bundle.
+
+## Usage
+
+```js
+import { prepare } from "speechready";
+
+const result = prepare(
+  "**Step 1:** run `npm install`. [pause 1s] Then read https://example.com/docs.",
+  { provider: "elevenlabs", model: "eleven_multilingual_v2", locale: "en-US" },
+);
+
+result.text     // 'Step 1: run npm install. <break time="1s" /> Then read example.com.'
+result.chunks   // the same text split under the model's request limit
+for (const change of result.changes) {
+  console.log(change.rule, JSON.stringify(change.before), "->", JSON.stringify(change.after));
+}
+```
+
+`provider` and `model` pick a profile; with neither, the `plain` profile renders pauses as punctuation and returns one chunk. Pass `profile: loadProfile("azure-neural")` to pick one directly and `limit` to override the request limit. `locale` is `en-US` by default; `tr-TR` turns on the Turkish rules.
+
+Everything a caller may want to decide differently is an option. The names are the same as in the Python package and in the shared fixtures, so they are snake_case; a name the package does not know throws.
+
+```js
+prepare(text, { provider: "openai", options: {
+  code_blocks: "mention",     // drop (default) | keep | mention: one sentence stands in for a fenced block
+  code_block_mention: null,   // what `mention` says; null = "See the code block." / "Kod bloğuna bakın."
+  tables: "rows",             // rows (default, each row a comma list) | drop
+  urls: "domain",             // domain (default) | drop | keep
+  emails: "keep",             // keep (default) | drop
+  emoji: "drop",              // drop (default) | keep
+  emphasis: "strip",          // strip (default) | intent: **bold** becomes an emphasis intent
+  list_numbers: "keep",       // keep (default, "1. Open the app") | drop
+  fractions: "spoken",        // spoken (default: "one half", "yarım") | division ("one over two", "bir bölü iki")
+  foreign_acronyms: true,     // Turkish: npm, css, sql read with English letter names (en pi em)
+  terminal_punctuation: ".",  // added to a block that ends without one
+}});
+```
+
+Stages are also exported on their own: `clean(text, options, recorder, locale)`, `verbalize(text, locale, recorder, options)`, `splitSentences(text, locale)`, `pack(sentences, limit, unit)`, and `Segmenter` for streaming token feeds:
+
+```js
+import { Segmenter } from "speechready";
+
+const seg = new Segmenter({ locale: "en-US" });
+for (const token of llmStream) {
+  for (const sentence of seg.push(token)) speak(sentence);   // released once the boundary is confirmed
+}
+for (const sentence of seg.flush()) speak(sentence);
+```
+
+Pronunciation lexicon, one file for every vendor:
+
+```js
+import { Lexicon, prepare } from "speechready";
+
+const lex = Lexicon.fromEntries([
+  { word: "Nginx", ipa: "ˈɛndʒɪnˈɛks", say: "engine X" },
+  { word: "SQL", say: "sequel" },
+  { word: "CLI", spell: true },
+]);
+prepare("Restart Nginx.", { provider: "azure", lexicon: lex }).text;
+// 'Restart <phoneme alphabet="ipa" ph="ˈɛndʒɪnˈɛks">Nginx</phoneme>.'
+prepare("Restart Nginx.", { provider: "openai", lexicon: lex }).text;
+// 'Restart engine X.'            (no phoneme tags on OpenAI, so the respelling is used)
+
+lex.toPls();          // PLS XML for ElevenLabs, Azure and Polly uploads
+lex.toGoogleJson();   // Google Cloud TTS custom pronunciations
+await Lexicon.load("lexicon.json");   // Node only; .yaml files need the optional `yaml` package
+```
+
+## Differences from the Python package
+
+The two packages produce the same text for every fixture in `spec/fixtures`. The API differs only where the languages do: keyword arguments are one options object, methods are camelCase (`limitFor`, `toPls`, `splitSentences`), `Lexicon.load` is async, and `prepare` takes a `Lexicon` or an entries array rather than a file path.
+
+## Development
+
+```
+cd js
+npm ci
+npm test        # builds, then runs every fixture in ../spec/fixtures plus the package tests
+```
+
+`npm run build` regenerates `src/profiles/data.ts` from `../spec/profiles` and `src/version.ts` from `package.json`, then compiles to `dist/`. All three are build outputs and are not committed.

package/dist/changes.d.ts

@@ -0,0 +1,14 @@
+/** The audit trail: one Change per rewrite the pipeline made. */
+export interface Change {
+    stage: string;
+    /** Dotted and stable, e.g. `clean.markdown.heading`. */
+    rule: string;
+    before: string;
+    after: string;
+    note: string;
+}
+/** Collects Changes; stages call `add` whenever they rewrite something. */
+export declare class Recorder {
+    changes: Change[];
+    add(stage: string, rule: string, before: string, after: string, note?: string): void;
+}

package/dist/changes.js

@@ -0,0 +1,10 @@
+/** The audit trail: one Change per rewrite the pipeline made. */
+/** Collects Changes; stages call `add` whenever they rewrite something. */
+export class Recorder {
+    changes = [];
+    add(stage, rule, before, after, note = "") {
+        if (before === after)
+            return;
+        this.changes.push({ stage, rule, before, after, note });
+    }
+}

package/dist/chunk.d.ts

@@ -0,0 +1,32 @@
+/** Stage 4: chunk. Sentence segmentation, streaming segmentation, and packing under a limit.
+ *
+ * Rules that keep prosody intact:
+ * - a newline is a hard boundary (blocks from the clean stage never merge);
+ * - a sentence ends at . ! ? … followed by whitespace and then an uppercase
+ *   letter, a digit, an opening quote or bracket, or a tag;
+ * - a known abbreviation, a single-letter initial, or a number before the full
+ *   stop never ends a sentence;
+ * - a sentence that is longer than the limit is split at clause punctuation,
+ *   then at spaces, and only as a last resort mid-word.
+ */
+/** Split text into sentences. Newlines are hard boundaries. Empty sentences are dropped. */
+export declare function splitSentences(text: string, locale?: string): string[];
+/** Streaming sentence segmenter for LLM token feeds.
+ *
+ * `push(text)` returns the sentences that are now complete; `flush()` returns
+ * whatever is left. A boundary is only confirmed once text after it arrives,
+ * so a sentence is never cut at "Dr." or "3.14" because the stream paused there.
+ */
+export declare class Segmenter {
+    locale: string;
+    maxBuffer: number | null;
+    private buffer;
+    constructor(init?: {
+        locale?: string;
+        maxBuffer?: number | null;
+    });
+    push(text: string): string[];
+    flush(): string[];
+}
+/** Greedily join sentences into chunks no larger than `limit`. null = one chunk. */
+export declare function pack(sentences: string[], limit: number | null | undefined, unit?: string, joiner?: string): string[];

package/dist/chunk.js

@@ -0,0 +1,195 @@
+/** Stage 4: chunk. Sentence segmentation, streaming segmentation, and packing under a limit.
+ *
+ * Rules that keep prosody intact:
+ * - a newline is a hard boundary (blocks from the clean stage never merge);
+ * - a sentence ends at . ! ? … followed by whitespace and then an uppercase
+ *   letter, a digit, an opening quote or bracket, or a tag;
+ * - a known abbreviation, a single-letter initial, or a number before the full
+ *   stop never ends a sentence;
+ * - a sentence that is longer than the limit is split at clause punctuation,
+ *   then at spaces, and only as a last resort mid-word.
+ */
+import { SENTINEL_OPEN } from "./intents.js";
+import { codePoints, finditer, fullmatch, isAlpha, isDigit, isUpper, re, rstrip, strip, utf8Length } from "./util.js";
+const ABBREVIATIONS = {
+    en: new Set(`mr mrs ms dr prof sr jr st mt ft vs etc e.g i.e cf no nos inc ltd co corp llc dept est approx
+    jan feb mar apr jun jul aug sep sept oct nov dec mon tue wed thu fri sat sun a.m p.m
+    u.s u.k e.u ph.d m.d b.a m.a b.sc m.sc fig vol pp ed eds rev gen col lt sgt capt cmdr hon
+    ave blvd rd hwy tel ext dept univ assn bros`.split(/\s+/)),
+    tr: new Set(`dr prof doç yrd av sn vb vs vd örn bkz yy t.c no sok cad apt mah alb yzb bşk müh
+    oca şub mar nis haz tem ağu eyl eki kas ara ltd şti a.ş san tic dk sa sn bk ör
+    hz sf s c çev der haz üniv fak böl hst bul uzm öğr yrd tel`.split(/\s+/)),
+};
+const END = re(String.raw `(?<end>[.!?…]+)(?<close>["'”’)\]»]*)(?<space>\s+)(?=(?<next>\S))`);
+const WORD_BEFORE = re(String.raw `(\S+)$`);
+const CLAUSE = re(String.raw `[,;:—–]\s+|\s+[-]\s+`);
+const DOTTED = re("[a-z](\\.[a-z])+");
+const LIST_MARKER = re(String.raw `\d{1,3}\.`);
+function lang(locale) {
+    return (locale || "en").split("-")[0].split("_")[0].toLowerCase();
+}
+function abbreviations(locale) {
+    return ABBREVIATIONS[lang(locale)] ?? new Set();
+}
+function isBoundary(text, m, abbrs) {
+    const g = m.groups ?? {};
+    const end = g.end ?? "";
+    const nxt = g.next ?? "";
+    const before = text.slice(0, m.index);
+    const wordM = WORD_BEFORE.exec(before);
+    WORD_BEFORE.lastIndex = 0;
+    const word = wordM ? wordM[1] : "";
+    const core = strip(word, "(\"'“”‘’[").toLowerCase();
+    if ((g.space ?? "").includes("\n"))
+        return true;
+    if (end.startsWith(".") && !end.includes("…") && !end.includes("!") && !end.includes("?")) {
+        if (abbrs.has(core) || abbrs.has(rstrip(core, ".")))
+            return false;
+        if (core.length === 1 && isAlpha(core))
+            return false; // initial: "J. R. R. Tolkien"
+        if (isDigit(core) && !isUpper(nxt[0]))
+            return false; // ordinal or list number: "3. madde"
+        if (fullmatch(DOTTED, core))
+            return false; // dotted abbreviation not in the list, e.g. "p.s"
+    }
+    const first = nxt[0];
+    if ("\"'“‘([«<".includes(first) || first === SENTINEL_OPEN)
+        return true;
+    if (isUpper(first) || isDigit(first))
+        return true;
+    if ("¿¡".includes(first))
+        return true;
+    return false;
+}
+/** Split text into sentences. Newlines are hard boundaries. Empty sentences are dropped. */
+export function splitSentences(text, locale = "en-US") {
+    const abbrs = abbreviations(locale);
+    const out = [];
+    for (const rawLine of text.split("\n")) {
+        const line = rawLine.trim();
+        if (!line)
+            continue;
+        let start = 0;
+        for (const m of finditer(END, line)) {
+            const g = m.groups ?? {};
+            const endAll = m.index + m[0].length;
+            const endClose = m.index + (g.end ?? "").length + (g.close ?? "").length;
+            if (fullmatch(LIST_MARKER, line.slice(start, endAll).trim()))
+                continue; // "1. Open the app": the number is the item's marker, not a sentence
+            if (isBoundary(line, m, abbrs)) {
+                out.push(line.slice(start, endClose).trim());
+                start = endAll;
+            }
+        }
+        const tail = line.slice(start).trim();
+        if (tail)
+            out.push(tail);
+    }
+    return out;
+}
+/** Streaming sentence segmenter for LLM token feeds.
+ *
+ * `push(text)` returns the sentences that are now complete; `flush()` returns
+ * whatever is left. A boundary is only confirmed once text after it arrives,
+ * so a sentence is never cut at "Dr." or "3.14" because the stream paused there.
+ */
+export class Segmenter {
+    locale;
+    maxBuffer;
+    buffer = "";
+    constructor(init = {}) {
+        this.locale = init.locale ?? "en-US";
+        this.maxBuffer = init.maxBuffer ?? null;
+    }
+    push(text) {
+        this.buffer += text;
+        const done = [];
+        for (;;) {
+            const nl = this.buffer.indexOf("\n");
+            if (nl < 0)
+                break;
+            const head = this.buffer.slice(0, nl);
+            this.buffer = this.buffer.slice(nl + 1);
+            done.push(...splitSentences(head, this.locale));
+        }
+        const sentences = splitSentences(this.buffer, this.locale);
+        if (sentences.length > 1) {
+            done.push(...sentences.slice(0, -1));
+            // keep the unconfirmed tail exactly as it stands (leading spaces dropped)
+            const idx = this.buffer.lastIndexOf(sentences[sentences.length - 1]);
+            this.buffer = idx >= 0 ? this.buffer.slice(idx) : this.buffer;
+        }
+        if (this.maxBuffer && this.buffer.length > this.maxBuffer) {
+            const cut = lastClauseCut(this.buffer, this.maxBuffer);
+            if (cut > 0) {
+                done.push(this.buffer.slice(0, cut).trim());
+                this.buffer = this.buffer.slice(cut).replace(/^\s+/, "");
+            }
+        }
+        return done.filter((s) => s);
+    }
+    flush() {
+        const rest = this.buffer;
+        this.buffer = "";
+        return splitSentences(rest, this.locale);
+    }
+}
+function lastClauseCut(text, limit) {
+    const window = text.slice(0, limit);
+    let best = -1;
+    for (const m of finditer(CLAUSE, window))
+        best = m.index + m[0].length;
+    if (best > 0)
+        return best;
+    const sp = window.lastIndexOf(" ");
+    return sp > 0 ? sp + 1 : limit;
+}
+function size(text, unit) {
+    return unit === "bytes" ? utf8Length(text) : codePoints(text);
+}
+/** Split one over-long sentence at clause marks, then spaces, then hard. */
+function splitLong(sentence, limit, unit) {
+    if (size(sentence, unit) <= limit)
+        return [sentence];
+    const pieces = [];
+    let rest = sentence;
+    while (size(rest, unit) > limit) {
+        let window = rest;
+        while (size(window, unit) > limit)
+            window = window.slice(0, Math.max(1, Math.floor(window.length * 0.9)));
+        let cut = -1;
+        for (const m of finditer(CLAUSE, window))
+            cut = m.index + m[0].length;
+        if (cut <= 0)
+            cut = window.lastIndexOf(" ") + 1;
+        if (cut <= 0)
+            cut = window.length;
+        pieces.push(rest.slice(0, cut).trim());
+        rest = rest.slice(cut).trim();
+    }
+    if (rest)
+        pieces.push(rest);
+    return pieces;
+}
+/** Greedily join sentences into chunks no larger than `limit`. null = one chunk. */
+export function pack(sentences, limit, unit = "chars", joiner = " ") {
+    if (limit === null || limit === undefined)
+        return sentences.length ? [sentences.join(joiner)] : [];
+    const chunks = [];
+    let current = "";
+    for (const s of sentences) {
+        for (const piece of splitLong(s, limit, unit)) {
+            const candidate = current ? current + joiner + piece : piece;
+            if (current && size(candidate, unit) > limit) {
+                chunks.push(current);
+                current = piece;
+            }
+            else {
+                current = candidate;
+            }
+        }
+    }
+    if (current)
+        chunks.push(current);
+    return chunks;
+}

package/dist/clean.d.ts

@@ -0,0 +1,41 @@
+/** Stage 1: clean. Turn markdown, code, emoji, URLs and invisible Unicode into plain prose.
+ *
+ * Block structure is preserved as one block per line: headings, paragraphs,
+ * list items and table rows each become a line, and a block that ends without
+ * terminal punctuation gets a full stop so the model pauses after it. Hard-wrapped
+ * lines inside a paragraph are joined with a space.
+ */
+import { Recorder } from "./changes.js";
+/** Everything a caller can change about how text is prepared. Defaults are what the evaluation ran with.
+ *
+ * Field names are the same as in the Python package and the shared fixtures (snake_case).
+ */
+export interface Options {
+    /** drop | keep | mention (one spoken sentence stands in for the block) */
+    code_blocks: "drop" | "keep" | "mention";
+    /** the sentence `mention` speaks; null = the locale's default */
+    code_block_mention: string | null;
+    tables: "rows" | "drop";
+    urls: "domain" | "drop" | "keep";
+    emails: "keep" | "drop";
+    emoji: "drop" | "keep";
+    emphasis: "strip" | "intent";
+    /** keep ("1. Open the app") | drop ("Open the app") */
+    list_numbers: "keep" | "drop";
+    strip_tags: string[];
+    terminal_punctuation: string;
+    html: "strip" | "keep";
+    /** spoken ("one half", "yarım") | division ("one over two", "bir bölü iki") */
+    fractions: "spoken" | "division";
+    /** Turkish: lowercase vowel-less tokens like npm, css read with English letter names */
+    foreign_acronyms: boolean;
+    extra: Record<string, unknown>;
+}
+export declare const DEFAULT_OPTIONS: Options;
+/** Fill in defaults. Unknown keys throw, so a camelCase typo is not silently ignored. */
+export declare function options(partial?: Partial<Options> | null): Options;
+/** Run the clean stage on `text`; returns the cleaned text and records changes.
+ *
+ * `locale` only picks the sentence that `{code_blocks: "mention"}` speaks.
+ */
+export declare function clean(text: string, opts?: Partial<Options> | null, recorder?: Recorder, locale?: string): string;