puzlink 0.1.0

package/src/lib/wordlist.ts

@@ -0,0 +1,162 @@
+import { Cromulence, loadWordlist, logProbToZipf } from "cromulence";
+import { PrefixDistribution, SuffixDistribution } from "./affixDistribution.js";
+import { LetterBitsets } from "./letterBitset.js";
+import { LetterDistribution } from "./letterDistribution.js";
+import { LogNum } from "./logNum.js";
+import { memoize } from "./memoize.js";
+
+/**
+ * Info about the words in a wordlist.
+ *
+ * We assume (as in `cromulence`) that words appearing in puzzles are
+ * distributed via Zipf frequency.
+ */
+export class Wordlist {
+  private cromulence: Cromulence;
+  /** A map from letter bitsets to words with that bitset. */
+  private bitsets: LetterBitsets;
+  /** The letter distribution of the wordlist. */
+  letters: LetterDistribution;
+  /** The prefix distribution of the wordlist. */
+  prefixes: PrefixDistribution;
+  /** The suffix distribution of the wordlist. */
+  suffixes: SuffixDistribution;
+
+  constructor(wordlist: Record<string, number>) {
+    this.cromulence = new Cromulence(wordlist);
+    this.bitsets = new LetterBitsets(Object.keys(wordlist));
+    this.letters = new LetterDistribution(Object.keys(wordlist));
+    this.prefixes = new PrefixDistribution(Object.keys(wordlist));
+    this.suffixes = new SuffixDistribution(Object.keys(wordlist));
+  }
+
+  static async download(): Promise<Wordlist> {
+    return new Wordlist(await loadWordlist());
+  }
+
+  /**
+   * For testing purposes: create a wordlist from an array of words, each
+   * equiprobable.
+   */
+  static from(words: string[]): Wordlist {
+    const logFrac = LogNum.fromFraction(1, words.length);
+    const wordlist: Record<string, LogNum> = {};
+    for (const word of words) {
+      wordlist[word] = wordlist[word] ? wordlist[word].add(logFrac) : logFrac;
+    }
+    return new Wordlist(
+      Object.fromEntries(
+        Object.entries(wordlist).map(([word, logProb]) => [
+          word,
+          logProbToZipf(logProb.toLog()),
+        ]),
+      ),
+    );
+  }
+
+  /** Apply a reducer to each word in the wordlist. */
+  reduce<T>(initial: T, reducer: (acc: T, slug: string, zipf: number) => T): T {
+    let result = initial;
+    for (const slug in this.cromulence.wordlist) {
+      const zipf = this.cromulence.wordlist[slug]!;
+      result = reducer(result, slug, zipf);
+    }
+    return result;
+  }
+
+  /**
+   * Get the log prob that a wordlist item, drawn uniformly at random,
+   * satisfies the given property. This is NOT weighted by zipf!
+   */
+  logProb(property: (slug: string) => boolean): LogNum {
+    // TODO: maybe tweak the weights here to get better results?
+    // if we do so, do the same thing in letterDistribution
+    const count = this.reduce(0, (acc, slug) => acc + (property(slug) ? 1 : 0));
+    const total = Object.keys(this.cromulence.wordlist).length;
+    return LogNum.fromFraction(count, total);
+  }
+
+  /** Returns true if the given slug is in the wordlist. */
+  isWord(
+    slug: string,
+    { threshold = 0 }: { threshold?: number } = {},
+  ): boolean {
+    return (this.cromulence.wordlist[slug] ?? -1000) > threshold;
+  }
+
+  /** Returns true if any of the given slugs are in the wordlist. */
+  hasWord(
+    slugs: string[],
+    { threshold = 0 }: { threshold?: number } = {},
+  ): boolean {
+    return slugs.some((slug) => this.isWord(slug, { threshold }));
+  }
+
+  /** Filters for slugs in the wordlist, sorted from most common to least. */
+  filterWords(
+    slugs: string[],
+    { threshold = 0 }: { threshold?: number } = {},
+  ): string[] {
+    return slugs
+      .map((slug) => [slug, this.cromulence.wordlist[slug]] as const)
+      .filter(
+        (t): t is [string, number] => t[1] !== undefined && t[1] > threshold,
+      )
+      .sort((a, b) => b[1] - a[1])
+      .map((t) => t[0]);
+  }
+
+  /**
+   * Filters for slugs in the wordlist under the given getter, sorted from most
+   * common to least.
+   */
+  filterWordsUnder<T>(
+    items: T[],
+    getSlug: (item: T) => string,
+    { threshold = 0 }: { threshold?: number } = {},
+  ): T[] {
+    return items
+      .map((item) => [item, this.cromulence.wordlist[getSlug(item)]] as const)
+      .filter((t): t is [T, number] => t[1] !== undefined && t[1] > threshold)
+      .sort((a, b) => b[1] - a[1])
+      .map((t) => t[0]);
+  }
+
+  /** Returns true if the given phrase is in the wordlist. */
+  isPhrase(phrase: string): boolean {
+    return this.cromulence.cromulence(phrase) > 0;
+  }
+
+  /** Returns the anagrams of a given slug, sorted from most common to least. */
+  anagrams(
+    slug: string,
+    {
+      loose = false,
+      threshold = 0,
+    }: { loose?: boolean; threshold?: number } = {},
+  ): string[] {
+    return this.bitsets
+      .get(slug)
+      .filter((word) => loose || word !== slug)
+      .map((word) => [word, this.cromulence.wordlist[word]!] as const)
+      .filter((t) => t[1] > threshold)
+      .sort((a, b) => b[1] - a[1])
+      .map((t) => t[0]);
+  }
+
+  /**
+   * Prob that, for two words, the first has a suffix equal to the second's
+   * prefix, of the given length.
+   */
+  @memoize()
+  probSharedAffix(length: number) {
+    const prefixes = this.prefixes.get(length);
+    const suffixes = this.suffixes.get(length);
+
+    return LogNum.sum(
+      Array.from(prefixes.entries(), ([prefix, prefixProb]) => {
+        return prefixProb.mul(suffixes.get(prefix));
+      }),
+    );
+  }
+}

package/src/linkers/index.ts

@@ -0,0 +1,56 @@
+import { answerLengthLogProbs } from "../data/answerLengths.js";
+import { featureLinkers } from "../features/index.js";
+import type { LinkOptions } from "../index.js";
+import { LengthDistribution } from "../lib/lengthDistribution.js";
+import { LogNum } from "../lib/logNum.js";
+import type { Wordlist } from "../lib/wordlist.js";
+import { indexingLinker } from "./indexing.js";
+import { lengthLinker } from "./length.js";
+import { otherLinker } from "./other.js";
+
+/**
+ * A PartialLink is the subset of Link that a Linker needs to return. We do
+ * some processing before it ends up being a Link. See Link for full details.
+ */
+export type PartialLink = {
+  /** A human-readable link name; defaults to the name of the linker. */
+  name?: string;
+  /**
+   * Log prob we'd expect to see this link.
+   *
+   * Note that this should describe the log prob of the *name* of the link.
+   * A link with name "two distinct length values" should have the log prob
+   * that the words have *any two* distinct lengths, and the description
+   * should report more specifically what those lengths are.
+   */
+  logProb: LogNum;
+  /** Any extra info to include in the link. Can be blank. */
+  description: readonly string[];
+};
+
+/**
+ * A Linker is a function that takes a list of slugs and returns PartialLinks.
+ */
+export type Linker = {
+  name: string;
+  eval: (slugs: string[], options: Required<LinkOptions>) => PartialLink[];
+};
+
+/** All linkers. */
+export function allLinkers(wordlist: Wordlist): Linker[] {
+  const lengthDist = LengthDistribution.from(answerLengthLogProbs);
+  return [
+    ...featureLinkers(wordlist),
+    indexingLinker(wordlist),
+    lengthLinker(lengthDist),
+    otherLinker(wordlist),
+  ];
+}
+
+/** For testing purposes. */
+export const testLinkOptions: Required<LinkOptions> = {
+  lazy: false,
+  limit: Infinity,
+  minFeatureRatio: 0,
+  ordered: true,
+};

package/src/linkers/indexing.ts

@@ -0,0 +1,194 @@
+import { LetterIndices } from "../lib/letterIndices.js";
+import { getArithmeticSequenceInfo, interval, ordinal } from "../lib/util.js";
+import type { Wordlist } from "../lib/wordlist.js";
+import type { Linker, PartialLink } from "./index.js";
+
+type Indexed = {
+  /** "the 2nd letters", "the diagonal", "the antidiagonal" */
+  indexText: string;
+  /** Result of indexing the kind of thing in `indexText`. */
+  indexed: string[];
+};
+
+function* indicesFor(slugs: string[], ordered?: boolean): Generator<Indexed> {
+  for (const i of interval(-10, 9)) {
+    const indexText =
+      i >= 0 ? `${ordinal(i + 1)} letters` : `${ordinal(i)} letters`;
+    const indexed = slugs.map((slug) => slug.at(i) ?? null);
+    if (indexed.every((c) => c !== null)) {
+      yield { indexText, indexed };
+    }
+  }
+  if (ordered) {
+    const indexed = slugs.map((slug, i) => slug.at(i) ?? null);
+    if (indexed.every((c) => c !== null)) {
+      yield { indexText: "diagonal", indexed };
+    }
+  }
+  if (ordered && slugs.every((slug) => slug.length >= slugs.length)) {
+    yield {
+      indexText: "antidiagonal",
+      indexed: slugs.map((slug, i) => slug.at(slugs.length - i - 1)!),
+    };
+  }
+}
+
+type Props = Indexed & {
+  slugs: string[];
+  wordlist: Wordlist;
+};
+
+function allEqual({ indexText, indexed, wordlist }: Props): PartialLink | null {
+  if (new Set(indexed).size !== 1) {
+    return null;
+  }
+  return {
+    name: `${indexText} are equal`,
+    logProb: wordlist.letters.probEqual(indexed.length),
+    description: [`${indexText} are ${indexed[0]!}`],
+  };
+}
+
+function almostEqual({
+  indexText,
+  indexed,
+  slugs,
+  wordlist,
+}: Props): PartialLink | null {
+  const indexedSet = new Set(indexed);
+  if (indexedSet.size !== 2) {
+    return null;
+  }
+  let [a, b] = indexedSet;
+  let aSlugIndices = indexed.flatMap((w, i) => (w === a ? [i] : []));
+  let bSlugIndices = indexed.flatMap((w, i) => (w === b ? [i] : []));
+  if (aSlugIndices.length > bSlugIndices.length) {
+    [aSlugIndices, bSlugIndices] = [bSlugIndices, aSlugIndices];
+    [a, b] = [b, a];
+  }
+  if (aSlugIndices.length !== 1) {
+    return null;
+  }
+  const [aSlugIndex] = aSlugIndices as [number];
+  return {
+    name: `${indexText} are almost equal`,
+    logProb: wordlist.letters.probAlmostEqual(indexed.length),
+    description: [
+      `'${slugs[aSlugIndex]!}' ${indexText} is '${a!}'`,
+      `others ${indexText} are '${b!}'`,
+    ],
+  };
+}
+
+function onlyTwo({
+  indexText,
+  indexed,
+  slugs,
+  wordlist,
+}: Props): PartialLink | null {
+  const indexedSet = new Set(indexed);
+  if (indexedSet.size !== 2) {
+    return null;
+  }
+  const [a, b] = indexedSet;
+  const aSlugIndices = indexed.flatMap((w, i) => (w === a ? [i] : []));
+  const bSlugIndices = indexed.flatMap((w, i) => (w === b ? [i] : []));
+  return {
+    name: `${indexText} have only two values`,
+    logProb: wordlist.letters.probTwoDistinct(indexed.length),
+    description: [
+      `${aSlugIndices.map((i) => `'${slugs[i]!}'`).join(", ")} ${indexText} are '${a!}'`,
+      `${bSlugIndices.map((i) => `'${slugs[i]!}'`).join(", ")} ${indexText} are '${b!}'`,
+    ],
+  };
+}
+
+function word({ indexText, indexed, wordlist }: Props): PartialLink | null {
+  // TODO: isPhrase?
+  if (!wordlist.isWord(indexed.join(""))) {
+    return null;
+  }
+  return {
+    name: `${indexText} are a word`,
+    logProb: wordlist.letters.probWord(indexed.length),
+    description: [`${indexText} are '${indexed.join("")}'`],
+  };
+}
+
+function anagram({ indexText, indexed, wordlist }: Props): PartialLink | null {
+  const anagrams = wordlist.anagrams(indexed.join(""));
+  if (anagrams.length === 0) {
+    return null;
+  }
+  return {
+    name: `${indexText} anagram to a word`,
+    logProb: wordlist.letters.probAnagram(indexed.length),
+    description: [`${indexText} anagram to ${anagrams.join(", ")}`],
+  };
+}
+
+function consecutive({
+  indexText,
+  indexed,
+  wordlist,
+}: Props): PartialLink | null {
+  const sorted = indexed
+    .slice()
+    .map((w) => w.charCodeAt(0))
+    .sort((a, b) => a - b);
+  if (getArithmeticSequenceInfo(sorted)?.step !== 1) {
+    return null;
+  }
+  return {
+    name: `${indexText} are consecutive`,
+    logProb: wordlist.letters.probConsecutive(indexed.length),
+    description: [`${indexText} are ${indexed.join(", ")}`],
+  };
+}
+
+function paired({
+  indexText,
+  indexed,
+  slugs,
+  wordlist,
+}: Props): PartialLink | null {
+  const byIndexed = LetterIndices.from(indexed.join(""));
+  const countSet = byIndexed.countSet();
+  if (countSet.size !== 1 || Array.from(countSet)[0] !== 2) {
+    return null;
+  }
+  return {
+    name: `${indexText} can be paired`,
+    logProb: wordlist.letters.probPaired(indexed.length),
+    description: Array.from(byIndexed.entries(), ([letter, slugIndices]) => {
+      return `${slugIndices.map((i) => `'${slugs[i]!}'`).join(", ")} ${indexText} are '${letter}'`;
+    }),
+  };
+}
+
+/** Links of the form "the nth indices are..." */
+export function indexingLinker(wordlist: Wordlist): Linker {
+  return {
+    name: "indexing linker",
+    eval: (slugs, { ordered }) => {
+      const indices = Array.from(indicesFor(slugs, ordered));
+      return indices.flatMap(({ indexText, indexed }) => {
+        const props = {
+          indexText,
+          indexed,
+          slugs,
+          wordlist,
+        };
+        return [
+          allEqual(props),
+          almostEqual(props),
+          onlyTwo(props),
+          ordered ? word(props) : null,
+          anagram(props),
+          consecutive(props),
+          paired(props),
+        ].filter((l): l is PartialLink => l !== null);
+      });
+    },
+  };
+}

package/src/linkers/length.ts

@@ -0,0 +1,122 @@
+import type { LengthDistribution } from "../lib/lengthDistribution.js";
+import { getArithmeticSequenceInfo } from "../lib/util.js";
+import type { Linker, PartialLink } from "./index.js";
+
+type Props = {
+  distribution: LengthDistribution;
+  lengthSet: Set<number>;
+  lengths: number[];
+  slugs: string[];
+};
+
+function allEqual({ distribution, lengthSet }: Props): PartialLink | null {
+  if (lengthSet.size !== 1) {
+    return null;
+  }
+  const [length] = Array.from(lengthSet) as [number];
+  return {
+    name: "all lengths equal",
+    logProb: distribution.probEqual(lengthSet.size),
+    description: [`all lengths are ${length.toString()}`],
+  };
+}
+
+function onlyTwo({
+  distribution,
+  lengthSet,
+  slugs,
+}: Props): PartialLink | null {
+  if (lengthSet.size !== 2) {
+    return null;
+  }
+  const [a, b] = Array.from(lengthSet) as [number, number];
+  const aLength = slugs.filter((w) => w.length === a);
+  const bLength = slugs.filter((w) => w.length === b);
+  const aLogProb = distribution.probEqual(aLength.length);
+  const bLogProb = distribution.probEqual(bLength.length);
+  return {
+    name: "only two lengths",
+    logProb: aLogProb.mul(bLogProb),
+    description: [
+      `length ${a.toString()}: ${aLength.join(", ")}`,
+      `length ${b.toString()}: ${bLength.join(", ")}`,
+    ],
+  };
+}
+
+function equalMod2({ distribution, lengths }: Props): PartialLink | null {
+  if (new Set(lengths.map((l) => l % 2)).size !== 1) {
+    return null;
+  }
+  const parity = lengths[0]! % 2;
+  return {
+    name: `all lengths are ${parity === 0 ? "even" : "odd"}`,
+    logProb: distribution.probEqualMod2(lengths.length),
+    description: [`all lengths are ${parity === 0 ? "even" : "odd"}`],
+  };
+}
+
+function equalMod3({ distribution, lengths }: Props): PartialLink | null {
+  if (new Set(lengths.map((l) => l % 3)).size !== 1) {
+    return null;
+  }
+  return {
+    name: "all lengths are equal mod 3",
+    logProb: distribution.probEqualMod3(lengths.length),
+    description: [`all lengths are equal mod 3`],
+  };
+}
+
+function consecutive({ distribution, lengths }: Props): PartialLink | null {
+  if (getArithmeticSequenceInfo(lengths)?.step !== 1) {
+    return null;
+  }
+  return {
+    name: "lengths are consecutive",
+    logProb: distribution.probConsecutive(lengths.length),
+    description: [`lengths are ${lengths.join(", ")}`],
+  };
+}
+
+function paired({ distribution, slugs }: Props): PartialLink | null {
+  const byLength = new Map<number, string[]>();
+  for (const slug of slugs) {
+    if (!byLength.has(slug.length)) {
+      byLength.set(slug.length, []);
+    }
+    byLength.get(slug.length)!.push(slug);
+  }
+  const lengthCounter = new Set(
+    Array.from(byLength.values(), (slugs) => slugs.length),
+  );
+  if (lengthCounter.size === 1 && Array.from(lengthCounter)[0] === 2) {
+    return {
+      name: "lengths can be paired",
+      logProb: distribution.probPaired(slugs.length),
+      description: Array.from(byLength.entries(), ([, slugs]) => {
+        return slugs.join(" and ");
+      }),
+    };
+  }
+  return null;
+}
+
+/** Length-based linker. */
+export function lengthLinker(distribution: LengthDistribution): Linker {
+  return {
+    name: "slug lengths",
+    eval: (slugs) => {
+      const lengths = slugs.map((w) => w.length).sort();
+      const lengthSet = new Set(lengths);
+      const props = { distribution, lengthSet, lengths, slugs };
+      return [
+        allEqual(props),
+        onlyTwo(props),
+        equalMod2(props),
+        equalMod3(props),
+        consecutive(props),
+        paired(props),
+      ].filter((l): l is PartialLink => l !== null);
+    },
+  };
+}

package/src/linkers/other.ts

@@ -0,0 +1,117 @@
+import { VOWELS } from "../lib/letterDistribution.js";
+import { LogNum } from "../lib/logNum.js";
+import { enumerate } from "../lib/util.js";
+import type { Wordlist } from "../lib/wordlist.js";
+import type { Linker, PartialLink } from "./index.js";
+
+type Props = {
+  slugs: string[];
+  wordlist: Wordlist;
+};
+
+function unusualLetters({ slugs, wordlist }: Props): PartialLink | null {
+  const all = Array.from(slugs.join(""));
+  const { high, low } = wordlist.letters.outliers(all);
+  if (high.length > 0 || low.length > 0) {
+    return {
+      name: "unusual letter distribution",
+      logProb: wordlist.letters.probUnordered(all),
+      description: [
+        ...(high.length > 0 ? [`over-represented: ${high.join(", ")}`] : []),
+        ...(low.length > 0 ? [`under-represented: ${low.join(", ")}`] : []),
+      ],
+    };
+  }
+  return null;
+}
+
+function equalVowelPattern(
+  start: boolean,
+  { slugs, wordlist }: Props,
+): PartialLink | null {
+  const minLength = Math.min(...slugs.map((s) => s.length));
+  const shortest = slugs.find((s) => s.length === minLength)!;
+  const pattern = Array.from(shortest, (letter) =>
+    VOWELS.includes(letter) ? "V" : "C",
+  ).join("");
+  const matchesVowelPattern = (other: string) => {
+    const offset = start ? 0 : other.length - minLength;
+    for (const [i, vc] of enumerate(pattern)) {
+      if (VOWELS.includes(other[offset + i]!) !== (vc === "V")) {
+        return false;
+      }
+    }
+    return true;
+  };
+  if (!slugs.every(matchesVowelPattern)) {
+    return null;
+  }
+  return {
+    name: `${start ? "start" : "end"} with the same vowel-consonant pattern`,
+    logProb: wordlist[start ? "prefixes" : "suffixes"].probEqualVowelPattern(
+      slugs.length,
+      minLength,
+    ),
+    description: [`all ${start ? "start" : "end"} with ${pattern}`],
+  };
+}
+
+function sharedAffixes({ slugs, wordlist }: Props): PartialLink | null {
+  const shared = new Map<string, { prefixOf: string; length: number }>();
+  for (const suffixOf of slugs) {
+    for (const prefixOf of slugs) {
+      if (prefixOf === suffixOf) {
+        continue;
+      }
+      let length = Math.min(prefixOf.length, suffixOf.length);
+      for (; length > 1; length--) {
+        if (prefixOf.slice(0, length) !== suffixOf.slice(-length)) {
+          continue;
+        }
+        const currentBest = shared.get(suffixOf)?.length ?? 0;
+        if (length > currentBest) {
+          shared.set(suffixOf, { prefixOf, length });
+        }
+      }
+    }
+  }
+
+  if (shared.size <= 1) {
+    return null;
+  }
+
+  return {
+    name: "multiple shared suffixes and prefixes",
+    // This is an underestimate because it assumes independence.
+    logProb: LogNum.prod(
+      Array.from(shared.values(), ({ length }) => {
+        return wordlist.probSharedAffix(length);
+      }),
+    ),
+    description: Array.from(
+      shared.entries(),
+      ([suffixOf, { prefixOf, length }]) =>
+        `${suffixOf.slice(0, -length)}${suffixOf.slice(-length).toUpperCase()} ${prefixOf
+          .slice(0, length)
+          .toUpperCase()}${prefixOf.slice(length)}`,
+    ),
+  };
+}
+
+// TODO at each index, there's exactly one repeated letter
+
+/** Other links. */
+export function otherLinker(wordlist: Wordlist): Linker {
+  return {
+    name: "other links",
+    eval: (slugs) => {
+      const props = { slugs, wordlist };
+      return [
+        unusualLetters(props),
+        equalVowelPattern(true, props),
+        equalVowelPattern(false, props),
+        sharedAffixes(props),
+      ].filter((l): l is PartialLink => l !== null);
+    },
+  };
+}

package/src/parse.ts

@@ -0,0 +1,20 @@
+/**
+ * Parse an input to a list of slugs.
+ *
+ * If the input has newlines, we split by newlines. Otherwise, if commas
+ * exist, we split by commas. Otherwise, we split by spaces.
+ */
+export function parse(words: string | readonly string[]): string[] {
+  if (typeof words === "string") {
+    if (words.includes("\n")) {
+      words = words.split("\n");
+    } else if (words.includes(",")) {
+      words = words.split(",");
+    } else {
+      words = words.split(" ");
+    }
+  }
+  return words
+    .map((w) => w.toLowerCase().replace(/[^a-z]/g, ""))
+    .filter((w) => w.length > 0);
+}