scrabble-solver 2.16.7 → 2.18.0

package/packages/dictionaries/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrabble-solver/dictionaries",
-  "version": "2.16.7",
+  "version": "2.18.0",
   "description": "Scrabble Solver 2 - Dictionaries",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -26,13 +26,14 @@
     "build": "tsc --project .",
     "clean": "rimraf build/",
     "clean:dictionaries": "node ./bin/remove-dictionaries.js",
-    "postbuild": "node ./bin/update-dictionaries.js"
+    "postbuild": "node ./bin/update-dictionaries.js",
+    "test": "bun test src"
   },
   "dependencies": {
-    "@kamilmielnik/trie": "^5.1.0",
-    "@scrabble-solver/logger": "^2.16.7",
-    "@scrabble-solver/types": "^2.16.7",
-    "@scrabble-solver/word-lists": "^2.16.7"
+    "@kamilmielnik/gaddag": "^2.0.0",
+    "@scrabble-solver/logger": "^2.18.0",
+    "@scrabble-solver/types": "^2.18.0",
+    "@scrabble-solver/word-lists": "^2.18.0"
   },
   "gitHead": "1e2d1ad49a288d4a44196d8e3336740da132c637"
 }

package/packages/dictionaries/src/Dictionaries.ts

@@ -1,4 +1,4 @@
-import { type Trie } from '@kamilmielnik/trie';
+import { type Gaddag } from '@kamilmielnik/gaddag';
 import { logger } from '@scrabble-solver/logger';
 import { Locale } from '@scrabble-solver/types';
 import fs from 'fs';
@@ -8,23 +8,23 @@ import { createAsyncProxy, downloadDictionary, LayeredCache } from './lib';
 import { type Cache } from './types';
 
 export class Dictionaries {
-  private readonly cache: Cache<Locale, Trie>;
+  private readonly cache: Cache<Locale, Gaddag>;
 
-  private readonly downloadDictionaryProxies: Record<Locale, () => Promise<Trie>>;
+  private readonly downloadDictionaryProxies: Record<Locale, () => Promise<Gaddag>>;
 
   constructor() {
     this.cache = new LayeredCache();
     this.downloadDictionaryProxies = Object.fromEntries(
       Object.values(Locale).map((locale) => [locale, createAsyncProxy(() => downloadDictionary(locale))]),
-    ) as Record<Locale, () => Promise<Trie>>;
+    ) as Record<Locale, () => Promise<Gaddag>>;
   }
 
-  public async get(locale: Locale): Promise<Trie> {
+  public async get(locale: Locale): Promise<Gaddag> {
     if (this.cache.has(locale)) {
-      const trie = await this.cache.get(locale);
+      const gaddag = await this.cache.get(locale);
 
-      if (trie) {
-        return trie;
+      if (gaddag) {
+        return gaddag;
       }
     }
 
@@ -47,12 +47,12 @@ export class Dictionaries {
     return Object.values(Locale).filter((locale) => this.cache.isStale(locale) !== false);
   }
 
-  private async updateDictionary(locale: Locale): Promise<Trie> {
+  private async updateDictionary(locale: Locale): Promise<Gaddag> {
     logger.info('Dictionaries - updateDictionary', { locale });
     fs.mkdirSync(OUTPUT_DIRECTORY, { recursive: true });
     const downloadDictionaryProxy = this.downloadDictionaryProxies[locale];
-    const trie = await downloadDictionaryProxy();
-    await this.cache.set(locale, trie);
-    return trie;
+    const gaddag = await downloadDictionaryProxy();
+    await this.cache.set(locale, gaddag);
+    return gaddag;
   }
 }

package/packages/dictionaries/src/lib/DiskCache.test.ts

@@ -0,0 +1,85 @@
+import { Gaddag } from '@kamilmielnik/gaddag';
+import { Locale } from '@scrabble-solver/types';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+import { DiskCache } from './DiskCache';
+import { getDictionaryFilepath, getLegacyDictionaryFilepath } from './getDictionaryFilepath';
+
+const LOCALE = Locale.EN_US;
+const WORDS = ['scrabble', 'solver', 'gaddag'];
+
+describe('DiskCache', () => {
+  let directory: string;
+  let cache: DiskCache;
+
+  beforeEach(() => {
+    directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dictionaries-test-'));
+    cache = new DiskCache(directory);
+  });
+
+  afterEach(() => {
+    fs.rmSync(directory, { force: true, recursive: true });
+  });
+
+  it('misses when there is no file', async () => {
+    expect(cache.has(LOCALE)).toBe(false);
+    expect(await cache.get(LOCALE)).toBeUndefined();
+    expect(cache.isStale(LOCALE)).toBeUndefined();
+    expect(cache.getLastModifiedTimestamp(LOCALE)).toBeUndefined();
+  });
+
+  it('round-trips a dictionary', async () => {
+    await cache.set(LOCALE, Gaddag.fromArray(WORDS));
+
+    expect(cache.has(LOCALE)).toBe(true);
+    expect(cache.isStale(LOCALE)).toBe(false);
+
+    const gaddag = await cache.get(LOCALE);
+
+    for (const word of WORDS) {
+      expect(gaddag?.has(word)).toBe(true);
+    }
+
+    expect(gaddag?.has('trie')).toBe(false);
+  });
+
+  it('treats a file written in an incompatible format as a cache miss and removes it', async () => {
+    const filepath = getDictionaryFilepath(LOCALE, directory);
+    fs.writeFileSync(filepath, 'serialized trie from a previous app version');
+
+    expect(cache.has(LOCALE)).toBe(true);
+    expect(await cache.get(LOCALE)).toBeUndefined();
+    expect(cache.has(LOCALE)).toBe(false);
+  });
+
+  it('treats a corrupted (truncated) file as a cache miss and removes it', async () => {
+    const serialized = Gaddag.fromArray(WORDS).serialize();
+    const filepath = getDictionaryFilepath(LOCALE, directory);
+    fs.writeFileSync(filepath, serialized.subarray(0, 10));
+
+    expect(await cache.get(LOCALE)).toBeUndefined();
+    expect(cache.has(LOCALE)).toBe(false);
+  });
+
+  it('removes the legacy pre-gaddag cache file when writing', async () => {
+    const legacyFilepath = getLegacyDictionaryFilepath(LOCALE, directory);
+    fs.writeFileSync(legacyFilepath, 'serialized trie');
+
+    await cache.set(LOCALE, Gaddag.fromArray(WORDS));
+
+    expect(fs.existsSync(legacyFilepath)).toBe(false);
+    expect(cache.has(LOCALE)).toBe(true);
+  });
+
+  it('reports files older than the stale threshold as stale', async () => {
+    await cache.set(LOCALE, Gaddag.fromArray(WORDS));
+
+    const filepath = getDictionaryFilepath(LOCALE, directory);
+    const twoDaysAgo = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000);
+    fs.utimesSync(filepath, twoDaysAgo, twoDaysAgo);
+
+    expect(cache.isStale(LOCALE)).toBe(true);
+  });
+});

package/packages/dictionaries/src/lib/DiskCache.ts

@@ -1,26 +1,44 @@
-import { Trie } from '@kamilmielnik/trie';
+import { Gaddag } from '@kamilmielnik/gaddag';
+import { logger } from '@scrabble-solver/logger';
 import { type Locale } from '@scrabble-solver/types';
 import fs from 'fs';
 
-import { CACHE_STALE_THRESHOLD } from '../constants';
+import { CACHE_STALE_THRESHOLD, OUTPUT_DIRECTORY } from '../constants';
 import type { Cache } from '../types';
 
-import { getDictionaryFilepath } from './getDictionaryFilepath';
+import { getDictionaryFilepath, getLegacyDictionaryFilepath } from './getDictionaryFilepath';
 
-export class DiskCache implements Cache<Locale, Trie> {
-  public async get(locale: Locale): Promise<Trie | undefined> {
+export class DiskCache implements Cache<Locale, Gaddag> {
+  private readonly directory: string;
+
+  constructor(directory: string = OUTPUT_DIRECTORY) {
+    this.directory = directory;
+  }
+
+  /**
+   * Returns undefined when the cached file cannot be deserialized (it was
+   * written by an app version with an incompatible format, or it is corrupted)
+   * so that callers re-download and re-serialize the dictionary.
+   */
+  public async get(locale: Locale): Promise<Gaddag | undefined> {
     if (!this.has(locale)) {
       return undefined;
     }
 
-    const filepath = getDictionaryFilepath(locale);
-    const serialized = await fs.promises.readFile(filepath, 'utf-8');
-    const trie = Trie.deserialize(serialized);
-    return trie;
+    const filepath = getDictionaryFilepath(locale, this.directory);
+
+    try {
+      const serialized = await fs.promises.readFile(filepath);
+      return Gaddag.deserialize(new Uint8Array(serialized.buffer, serialized.byteOffset, serialized.byteLength));
+    } catch (error) {
+      logger.warn('DiskCache - incompatible or corrupted dictionary file, it will be rebuilt', { error, locale });
+      await fs.promises.rm(filepath, { force: true });
+      return undefined;
+    }
   }
 
   public getLastModifiedTimestamp(locale: Locale): number | undefined {
-    const filepath = getDictionaryFilepath(locale);
+    const filepath = getDictionaryFilepath(locale, this.directory);
 
     if (!fs.existsSync(filepath)) {
       return undefined;
@@ -31,7 +49,7 @@ export class DiskCache implements Cache<Locale, Trie> {
   }
 
   public has(locale: Locale): boolean {
-    const filepath = getDictionaryFilepath(locale);
+    const filepath = getDictionaryFilepath(locale, this.directory);
     return fs.existsSync(filepath);
   }
 
@@ -50,8 +68,10 @@ export class DiskCache implements Cache<Locale, Trie> {
     return timeSinceModification > CACHE_STALE_THRESHOLD;
   }
 
-  public async set(locale: Locale, trie: Trie): Promise<void> {
-    const filepath = getDictionaryFilepath(locale);
-    await fs.promises.writeFile(filepath, trie.serialize());
+  public async set(locale: Locale, gaddag: Gaddag): Promise<void> {
+    const filepath = getDictionaryFilepath(locale, this.directory);
+    await fs.promises.writeFile(filepath, gaddag.serialize());
+    // Serialized-trie cache from before the GADDAG migration - remove this in #437
+    await fs.promises.rm(getLegacyDictionaryFilepath(locale, this.directory), { force: true });
   }
 }

package/packages/dictionaries/src/lib/LayeredCache.ts

@@ -1,4 +1,4 @@
-import { type Trie } from '@kamilmielnik/trie';
+import { type Gaddag } from '@kamilmielnik/gaddag';
 import { type Locale } from '@scrabble-solver/types';
 
 import type { Cache } from '../types';
@@ -7,10 +7,10 @@ import { createCacheTimestampComparator } from './createCacheTimestampComparator
 import { DiskCache } from './DiskCache';
 import { MemoryCache } from './MemoryCache';
 
-export class LayeredCache implements Cache<Locale, Trie> {
+export class LayeredCache implements Cache<Locale, Gaddag> {
   private readonly layers = [new MemoryCache(), new DiskCache()];
 
-  public async get(locale: Locale): Promise<Trie | undefined> {
+  public async get(locale: Locale): Promise<Gaddag | undefined> {
     const cache = this.getLastModifiedLayer(locale);
 
     if (!cache) {
@@ -53,13 +53,13 @@ export class LayeredCache implements Cache<Locale, Trie> {
     return false;
   }
 
-  public async set(locale: Locale, trie: Trie): Promise<void> {
+  public async set(locale: Locale, gaddag: Gaddag): Promise<void> {
     const [memoryCache, diskCache] = this.layers;
-    await diskCache.set(locale, trie);
-    await memoryCache.set(locale, trie);
+    await diskCache.set(locale, gaddag);
+    await memoryCache.set(locale, gaddag);
   }
 
-  private getLastModifiedLayer(locale: Locale): Cache<Locale, Trie> | undefined {
+  private getLastModifiedLayer(locale: Locale): Cache<Locale, Gaddag> | undefined {
     const layers = this.layers.filter((cache) => cache.has(locale));
     const [cached] = [...layers].sort(createCacheTimestampComparator(locale));
     return cached;

package/packages/dictionaries/src/lib/MemoryCache.ts

@@ -1,15 +1,15 @@
-import { type Trie } from '@kamilmielnik/trie';
+import { type Gaddag } from '@kamilmielnik/gaddag';
 import { type Locale } from '@scrabble-solver/types';
 
 import { CACHE_STALE_THRESHOLD } from '../constants';
 import type { Cache } from '../types';
 
-export class MemoryCache implements Cache<Locale, Trie> {
-  private readonly cache: Partial<Record<Locale, Trie>> = {};
+export class MemoryCache implements Cache<Locale, Gaddag> {
+  private readonly cache: Partial<Record<Locale, Gaddag>> = {};
 
   private readonly cacheTimestamps: Partial<Record<Locale, number>> = {};
 
-  public get(locale: Locale): Promise<Trie | undefined> {
+  public get(locale: Locale): Promise<Gaddag | undefined> {
     return Promise.resolve(this.cache[locale]);
   }
 
@@ -32,9 +32,9 @@ export class MemoryCache implements Cache<Locale, Trie> {
     return timeSinceModification > CACHE_STALE_THRESHOLD;
   }
 
-  public set(locale: Locale, trie: Trie): Promise<void> {
+  public set(locale: Locale, gaddag: Gaddag): Promise<void> {
     this.cacheTimestamps[locale] = Date.now();
-    this.cache[locale] = trie;
+    this.cache[locale] = gaddag;
     return Promise.resolve();
   }
 }

package/packages/dictionaries/src/lib/createCacheTimestampComparator.ts

@@ -1,10 +1,10 @@
-import { type Trie } from '@kamilmielnik/trie';
+import { type Gaddag } from '@kamilmielnik/gaddag';
 import { type Locale } from '@scrabble-solver/types';
 
 import type { Cache } from '../types';
 
 export const createCacheTimestampComparator = (locale: Locale) => {
-  return (a: Cache<Locale, Trie>, b: Cache<Locale, Trie>): number => {
+  return (a: Cache<Locale, Gaddag>, b: Cache<Locale, Gaddag>): number => {
     const aTimestamp = a.getLastModifiedTimestamp(locale);
     const bTimestamp = b.getLastModifiedTimestamp(locale);
 

package/packages/dictionaries/src/lib/downloadDictionary.ts

@@ -1,12 +1,13 @@
-import { Trie } from '@kamilmielnik/trie';
+import { Gaddag } from '@kamilmielnik/gaddag';
 import { logger } from '@scrabble-solver/logger';
 import { type Locale } from '@scrabble-solver/types';
 import { getWordList } from '@scrabble-solver/word-lists';
 
-export const downloadDictionary = async (locale: Locale): Promise<Trie> => {
+export const downloadDictionary = async (locale: Locale): Promise<Gaddag> => {
   logger.info('downloadDictionary', { locale });
   const words = await getWordList(locale);
   logger.info('downloadDictionary - success', { locale });
-  const trie = Trie.fromArray(words);
-  return trie;
+  const gaddag = Gaddag.fromArray(words);
+  logger.info('downloadDictionary - gaddag built', { locale });
+  return gaddag;
 };

package/packages/dictionaries/src/lib/getDictionaryFilepath.ts

@@ -3,6 +3,11 @@ import path from 'path';
 
 import { OUTPUT_DIRECTORY } from '../constants';
 
-export const getDictionaryFilepath = (locale: Locale): string => {
-  return path.resolve(OUTPUT_DIRECTORY, `${locale}.txt`);
+export const getDictionaryFilepath = (locale: Locale, directory: string = OUTPUT_DIRECTORY): string => {
+  return path.resolve(directory, `${locale}.gaddag`);
+};
+
+/** Filepath of the serialized-trie cache used before the GADDAG migration. Remove in #437. */
+export const getLegacyDictionaryFilepath = (locale: Locale, directory: string = OUTPUT_DIRECTORY): string => {
+  return path.resolve(directory, `${locale}.txt`);
 };

package/packages/gaddag/build/Gaddag.d.ts

@@ -0,0 +1,39 @@
+/**
+ * A GADDAG (Gordon, 1994) stored as flat typed arrays for speed and compact serialization.
+ *
+ * For every word `w` and every split `1 <= s <= |w|` the automaton accepts
+ * `reverse(w[0..s)) + ◇ + w[s..)` (the separator is omitted when `s === |w|`).
+ *
+ * States are identified by "refs": `ref = (firstArcIndex << 1) | isWordEnd`.
+ * A `firstArcIndex` of 0 means the state has no outgoing arcs; ref 0 means "no such state".
+ * Arcs of a state are contiguous; the last one is marked with {@link LAST_ARC_FLAG} in its label.
+ * An arc label stores the letter index (1..63, 0 = separator) in its low 6 bits.
+ */
+export declare class Gaddag {
+    /** Arc labels: letter index | LAST_ARC_FLAG. Index 0 is an unused sentinel. */
+    readonly arcLabels: Uint8Array;
+    /** Arc targets: encoded state refs. Index 0 is an unused sentinel. */
+    readonly arcTargets: Int32Array;
+    /** Ref of the root state. */
+    readonly rootRef: number;
+    /** Code point of each letter index (position 0 holds the code point of letter 1). */
+    readonly charCodes: Int32Array;
+    private readonly letterByCharCode;
+    constructor(arcLabels: Uint8Array, arcTargets: Int32Array, rootRef: number, charCodes: Int32Array);
+    static deserialize(bytes: Uint8Array): Gaddag;
+    serialize(): Uint8Array;
+    /** Number of arcs (including the unused sentinel at index 0). */
+    get arcsCount(): number;
+    /**
+     * Follows the arc labeled with `letter` from the state `ref` points at.
+     * Returns the target ref, or 0 when there is no such arc.
+     *
+     * A state's arcs are stored in ascending letter order, so the scan stops as
+     * soon as it passes the wanted letter.
+     */
+    getArc(ref: number, letter: number): number;
+    /** Maps a code point to its letter index, or -1 when the character is not in the alphabet. */
+    getLetter(charCode: number): number;
+    has(word: string): boolean;
+    hasPrefix(prefix: string): boolean;
+}

package/packages/gaddag/build/Gaddag.js

@@ -0,0 +1,128 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Gaddag = void 0;
+/* eslint-disable no-bitwise */
+const constants_1 = require("./constants");
+/**
+ * A GADDAG (Gordon, 1994) stored as flat typed arrays for speed and compact serialization.
+ *
+ * For every word `w` and every split `1 <= s <= |w|` the automaton accepts
+ * `reverse(w[0..s)) + ◇ + w[s..)` (the separator is omitted when `s === |w|`).
+ *
+ * States are identified by "refs": `ref = (firstArcIndex << 1) | isWordEnd`.
+ * A `firstArcIndex` of 0 means the state has no outgoing arcs; ref 0 means "no such state".
+ * Arcs of a state are contiguous; the last one is marked with {@link LAST_ARC_FLAG} in its label.
+ * An arc label stores the letter index (1..63, 0 = separator) in its low 6 bits.
+ */
+class Gaddag {
+    constructor(arcLabels, arcTargets, rootRef, charCodes) {
+        this.arcLabels = arcLabels;
+        this.arcTargets = arcTargets;
+        this.rootRef = rootRef;
+        this.charCodes = charCodes;
+        this.letterByCharCode = new Map();
+        for (let index = 0; index < charCodes.length; ++index) {
+            this.letterByCharCode.set(charCodes[index], index + 1);
+        }
+    }
+    static deserialize(bytes) {
+        // Note: an explicit copy (not .slice()) — Buffer.prototype.slice returns a
+        // view that would keep the misaligned byteOffset.
+        const aligned = bytes.byteOffset % 4 === 0 ? bytes : new Uint8Array(bytes);
+        const header = new Int32Array(aligned.buffer, aligned.byteOffset, 4);
+        if (header[0] !== constants_1.MAGIC) {
+            throw new Error('Invalid Gaddag data');
+        }
+        const letterCount = header[1];
+        const arcCount = header[2];
+        const rootRef = header[3];
+        const charCodes = new Int32Array(aligned.buffer, aligned.byteOffset + 16, letterCount);
+        const arcTargets = new Int32Array(aligned.buffer, aligned.byteOffset + 16 + 4 * letterCount, arcCount);
+        const arcLabels = new Uint8Array(aligned.buffer, aligned.byteOffset + 16 + 4 * (letterCount + arcCount), arcCount);
+        return new Gaddag(arcLabels, arcTargets, rootRef, charCodes);
+    }
+    serialize() {
+        const letterCount = this.charCodes.length;
+        const arcCount = this.arcTargets.length;
+        const bytes = new Uint8Array(16 + 4 * (letterCount + arcCount) + arcCount);
+        const header = new Int32Array(bytes.buffer, 0, 4);
+        header[0] = constants_1.MAGIC;
+        header[1] = letterCount;
+        header[2] = arcCount;
+        header[3] = this.rootRef;
+        new Int32Array(bytes.buffer, 16, letterCount).set(this.charCodes);
+        new Int32Array(bytes.buffer, 16 + 4 * letterCount, arcCount).set(this.arcTargets);
+        bytes.set(this.arcLabels, 16 + 4 * (letterCount + arcCount));
+        return bytes;
+    }
+    /** Number of arcs (including the unused sentinel at index 0). */
+    get arcsCount() {
+        return this.arcTargets.length;
+    }
+    /**
+     * Follows the arc labeled with `letter` from the state `ref` points at.
+     * Returns the target ref, or 0 when there is no such arc.
+     *
+     * A state's arcs are stored in ascending letter order, so the scan stops as
+     * soon as it passes the wanted letter.
+     */
+    getArc(ref, letter) {
+        let index = ref >>> 1;
+        if (index === 0) {
+            return 0;
+        }
+        const labels = this.arcLabels;
+        for (;;) {
+            const label = labels[index];
+            const arcLetter = label & constants_1.LETTER_MASK;
+            if (arcLetter === letter) {
+                return this.arcTargets[index];
+            }
+            if (arcLetter > letter || label >= constants_1.LAST_ARC_FLAG) {
+                return 0;
+            }
+            ++index;
+        }
+    }
+    /** Maps a code point to its letter index, or -1 when the character is not in the alphabet. */
+    getLetter(charCode) {
+        return this.letterByCharCode.get(charCode) ?? -1;
+    }
+    has(word) {
+        const length = word.length;
+        if (length === 0) {
+            return false;
+        }
+        let ref = this.rootRef;
+        for (let index = length - 1; index >= 0; --index) {
+            const letter = this.getLetter(word.charCodeAt(index));
+            if (letter === -1) {
+                return false;
+            }
+            ref = this.getArc(ref, letter);
+            if (ref === 0) {
+                return false;
+            }
+        }
+        return (ref & 1) === 1;
+    }
+    hasPrefix(prefix) {
+        const length = prefix.length;
+        if (length === 0) {
+            return true;
+        }
+        let ref = this.rootRef;
+        for (let index = length - 1; index >= 0; --index) {
+            const letter = this.getLetter(prefix.charCodeAt(index));
+            if (letter === -1) {
+                return false;
+            }
+            ref = this.getArc(ref, letter);
+            if (ref === 0) {
+                return false;
+            }
+        }
+        return (ref & 1) === 1 || this.getArc(ref, constants_1.SEPARATOR) !== 0;
+    }
+}
+exports.Gaddag = Gaddag;

package/packages/gaddag/build/buildGaddag.d.ts

@@ -0,0 +1,10 @@
+import { Gaddag } from './Gaddag';
+/**
+ * Builds a minimal GADDAG from a word list.
+ *
+ * The construction generates every GADDAG sequence (`reverse(prefix) [+ ◇ + suffix]`)
+ * as a compact `(wordIndex << 6) | splitIndex` integer, orders them with an in-place
+ * MSD radix sort, and feeds them to an incremental minimal-automaton builder
+ * (Daciuk et al., 2000) backed by typed arrays and an open-addressing state registry.
+ */
+export declare const buildGaddag: (words: string[]) => Gaddag;