puzlink 0.1.0

package/src/data/morse.ts

@@ -0,0 +1,28 @@
+export const morseLetter: Record<string, string> = {
+  a: ".-",
+  b: "-...",
+  c: "-.-.",
+  d: "-..",
+  e: ".",
+  f: "..-.",
+  g: "--.",
+  h: "....",
+  i: "..",
+  j: ".---",
+  k: "-.-",
+  l: ".-..",
+  m: "--",
+  n: "-.",
+  o: "---",
+  p: ".--.",
+  q: "--.-",
+  r: ".-.",
+  s: "...",
+  t: "-",
+  u: "..-",
+  v: "...-",
+  w: ".--",
+  x: "-..-",
+  y: "-.--",
+  z: "--..",
+};

package/src/data/scrabble.ts

@@ -0,0 +1,28 @@
+export const scrabbleLetterScore: Record<string, number> = {
+  e: 1,
+  a: 1,
+  i: 1,
+  o: 1,
+  n: 1,
+  r: 1,
+  t: 1,
+  l: 1,
+  s: 1,
+  u: 1,
+  d: 2,
+  g: 2,
+  b: 3,
+  c: 3,
+  m: 3,
+  p: 3,
+  f: 4,
+  h: 4,
+  v: 4,
+  w: 4,
+  y: 4,
+  k: 5,
+  j: 8,
+  x: 8,
+  q: 10,
+  z: 10,
+};

package/src/features/index.ts

@@ -0,0 +1,120 @@
+import { LetterIndices } from "../lib/letterIndices.js";
+import { LogNum } from "../lib/logNum.js";
+import { Wordlist } from "../lib/wordlist.js";
+import type { Linker } from "../linkers/index.js";
+import { letterCountFeatures } from "./letterCount.js";
+import { letterSequenceFeatures } from "./letterSequence.js";
+import { KnownLogProbs } from "./logProbCache.js";
+import { otherFeatures } from "./other.js";
+import { substringFeatures } from "./substring.js";
+import { wordplayFeatures } from "./wordplay.js";
+
+type Props = {
+  letterIndices: LetterIndices;
+  wordlist: Wordlist;
+};
+
+/**
+ * A Feature is a property that a slug can have.
+ *
+ * Features should be as specific as possible, like "can prepend T to get a
+ * word", rather than "can prepend a letter to get a word". Instead, use
+ * linkers to get more general features.
+ */
+export type Feature = {
+  /** The name of the feature; will be used for the linker name. */
+  name: string;
+  /**
+   * If the `slug` has the feature, returns a description with `slug` as the
+   * subject. This is an elaboration on the feature name.
+   */
+  property: (slug: string, props: Props) => string | null;
+};
+
+function getProps(wordlist: Wordlist, slug: string): Props {
+  return { letterIndices: LetterIndices.from(slug), wordlist };
+}
+
+/**
+ * Create a binomial linker for a given feature. A binomial link is the
+ * probability that at least k or at most k out of n words share the feature,
+ * whichever is less.
+ */
+function featureLinker(
+  wordlist: Wordlist,
+  { name, property }: Feature,
+): Linker | null {
+  let featureLogProb = KnownLogProbs.get(name, () => {
+    return wordlist.logProb(
+      (word) => property(word, getProps(wordlist, word)) !== null,
+    );
+  });
+  if (featureLogProb.toLog() === -Infinity) {
+    // We can't meaningfully make linkers out of zero-probability things,
+    // so just set it to something very small.
+    featureLogProb = LogNum.fromExp(-10);
+    return null;
+  }
+
+  return {
+    name,
+    eval: (slugs, options) => {
+      const description = slugs.flatMap((word) => {
+        const result = property(word, getProps(wordlist, word));
+        return result ? [result] : [];
+      });
+      if (
+        description.length !== 0 &&
+        description.length < options.minFeatureRatio * slugs.length
+      ) {
+        return [];
+      }
+      const logProb = LogNum.binomialPValue(
+        description.length,
+        slugs.length,
+        featureLogProb,
+      );
+      return [
+        {
+          name: `${description.length.toString()}/${slugs.length.toString()} ${name}`,
+          description,
+          logProb,
+        },
+      ];
+    },
+  };
+}
+
+/** Feature-based linkers. */
+export function featureLinkers(wordlist: Wordlist): Linker[] {
+  return [
+    ...letterCountFeatures(),
+    ...letterSequenceFeatures(),
+    ...otherFeatures(),
+    ...substringFeatures(),
+    ...wordplayFeatures(),
+  ].flatMap((feature) => {
+    const linker = featureLinker(wordlist, feature);
+    return linker ? [linker] : [];
+  });
+}
+
+/**
+ * For testing purposes. Takes a list of features, and returns
+ * a function that takes a slug and returns all features it satisfies.
+ */
+export function makeFeatureGetter(
+  features: Feature[],
+  wordlist: Wordlist,
+): (slug: string) => Record<string, string> {
+  return (slug) => {
+    const properties: Record<string, string> = {};
+    for (const feature of features) {
+      const property = feature.property(slug, getProps(wordlist, slug));
+      if (property) {
+        properties[feature.name] = property;
+      }
+    }
+    return properties;
+  };
+}

package/src/features/letterCount.ts

@@ -0,0 +1,174 @@
+import { Counter } from "../lib/counter.js";
+import { CONSONANTS, LETTERS, VOWELS } from "../lib/letterDistribution.js";
+import {
+  capitalizeAt,
+  getArithmeticSequenceInfo,
+  interval,
+  mapProduct,
+  windows,
+} from "../lib/util.js";
+import type { Feature } from "./index.js";
+
+function withTimes(letter: string, times: number, strict: boolean): Feature {
+  return {
+    name: strict
+      ? `has ${times.toString()} ${letter}`
+      : `has at least ${times.toString()} ${letter}`,
+    property: (slug, { letterIndices }) => {
+      const starts = letterIndices.get(letter);
+      if (strict ? starts.length !== times : starts.length < times) {
+        return null;
+      }
+      return capitalizeAt(slug, starts);
+    },
+  };
+}
+
+function uniqueOf(
+  kind: { name: string; letters: string },
+  times: number,
+): Feature {
+  return {
+    name: `has ${times.toString()} unique ${kind.name}`,
+    property: (slug, { letterIndices }) => {
+      const unique = letterIndices
+        .filterKeys((letter) => kind.letters.includes(letter))
+        .sort()
+        .join("");
+      if (unique.length !== times) {
+        return null;
+      }
+      return `${slug} ${kind.name}: ${unique}`;
+    },
+  };
+}
+
+function nGramRepeatsTimes(
+  kind: { name: string; n: number },
+  count: number,
+  repeats: number,
+  strict: boolean,
+): Feature {
+  return {
+    name: strict
+      ? `has ${count.toString()} ${kind.name}, each repeating ${repeats.toString()} times`
+      : `has ${count.toString()} ${kind.name}, each repeating at least ${repeats.toString()} times`,
+    property: (slug) => {
+      const counts = Counter.from(
+        Array.from(windows(slug, kind.n), (w) => w.join("")),
+      );
+      const nGrams = counts.filterKeys((_, count) =>
+        strict ? count === repeats : count >= repeats,
+      );
+      if (nGrams.length !== count) {
+        return null;
+      }
+      return `${slug}: ${nGrams.join(", ")}`;
+    },
+  };
+}
+
+function repeatedOf(kind: { name: string; letters: string }): Feature {
+  return {
+    name: `has repeated ${kind.name}`,
+    property: (slug, { letterIndices }) => {
+      const repeated = letterIndices.filterKeys(
+        (letter, indices) =>
+          kind.letters.includes(letter) && indices.length >= 2,
+      );
+      return repeated.length > 0 ? `${slug}: ${repeated.join(", ")}` : null;
+    },
+  };
+}
+
+function equalCounts(): Feature {
+  return {
+    name: "has equal letter counts",
+    property: (slug, { letterIndices }) => {
+      const countSet = letterIndices.countSet();
+      return countSet.size === 1
+        ? `${slug} letter counts are all ${Array.from(countSet)[0]!.toString()}`
+        : null;
+    },
+  };
+}
+
+function twoCounts(): Feature {
+  return {
+    name: "has one of two letter counts",
+    property: (slug, { letterIndices }) => {
+      const countSet = letterIndices.countSet();
+      if (countSet.size !== 2) {
+        return null;
+      }
+      const [a, b] = Array.from(countSet) as [number, number];
+      const aLetters = letterIndices.filterKeys(
+        (_, indices) => indices.length === a,
+      );
+      const bLetters = letterIndices.filterKeys(
+        (_, indices) => indices.length === b,
+      );
+      return `${slug} letter counts are ${a.toString()} (${aLetters.sort().join("")}) or ${b.toString()} (${bLetters.sort().join("")})`;
+    },
+  };
+}
+
+function arithmeticSequenceCounts(): Feature {
+  return {
+    name: "has letter counts in arithmetic sequence",
+    property: (slug, { letterIndices }) => {
+      const sortedCounts = Array.from(letterIndices.counts()).sort(
+        ([, a], [, b]) => a - b,
+      );
+      if (!getArithmeticSequenceInfo(sortedCounts.map(([, c]) => c))) {
+        return null;
+      }
+      const letters = sortedCounts.map(([l]) => l).join(", ");
+      const counts = sortedCounts.map(([, c]) => c).join(", ");
+      return `${slug} letter counts of ${letters} are ${counts}`;
+    },
+  };
+}
+
+/**
+ * Features for letter counts: things we can remark solely based on the
+ * histogram of letters/bigrams/trigrams in the slug.
+ */
+export function letterCountFeatures(): Feature[] {
+  return [
+    ...mapProduct(withTimes, LETTERS, interval(1, 5), [true, false]),
+    ...mapProduct(
+      uniqueOf,
+      [{ name: "vowels", letters: VOWELS }],
+      interval(0, 5),
+    ),
+    ...mapProduct(
+      uniqueOf,
+      [{ name: "consonants", letters: CONSONANTS }],
+      interval(1, 15),
+    ),
+    ...mapProduct(
+      uniqueOf,
+      [{ name: "letters", letters: LETTERS }],
+      interval(1, 26),
+    ),
+    ...mapProduct(
+      nGramRepeatsTimes,
+      [
+        { name: "letters", n: 1 },
+        { name: "bigrams", n: 2 },
+        { name: "trigrams", n: 3 },
+      ],
+      interval(1, 5),
+      interval(2, 4),
+      [true, false],
+    ),
+    ...mapProduct(repeatedOf, [
+      { name: "vowels", letters: VOWELS },
+      { name: "consonants", letters: CONSONANTS },
+    ]),
+    equalCounts(),
+    twoCounts(),
+    arithmeticSequenceCounts(),
+  ];
+}

package/src/features/letterSequence.ts

@@ -0,0 +1,222 @@
+import { CONSONANTS, LETTERS, VOWELS } from "../lib/letterDistribution.js";
+import {
+  enumerate,
+  interval,
+  mapProduct,
+  printIndexSlug,
+} from "../lib/util.js";
+import type { Feature } from "./index.js";
+
+// TODO: make the printed property names better here; some of these should be
+// more than just the raw index. evaluate after looking at integration tests
+
+function equalWithDistanceTimes(
+  letter: string,
+  distance: number,
+  times: number,
+): Feature {
+  return {
+    name: `has ${letter} with ${distance.toString()} letters between, ${times.toString()} times`,
+    property: (slug) => {
+      const starts = interval(0, slug.length - distance - 1).filter((i) => {
+        return slug[i] === letter && slug[i + distance + 1] === letter;
+      });
+      if (starts.length !== times) {
+        return null;
+      }
+      return printIndexSlug(
+        slug,
+        starts.flatMap((i) => [i, i + distance + 1]),
+      );
+    },
+  };
+}
+
+function unequalWithDistance(a: string, b: string, distance: number): Feature {
+  return {
+    name: `has ${a} and ${b} with ${distance.toString()} letters between`,
+    property: (slug) => {
+      if (a === b) {
+        return null;
+      }
+      const starts = interval(0, slug.length - distance - 1).filter((i) => {
+        return slug[i] === a && slug[i + distance + 1] === b;
+      });
+      if (starts.length === 0) {
+        return null;
+      }
+      return printIndexSlug(
+        slug,
+        starts.flatMap((i) => [i, i + distance + 1]),
+      );
+    },
+  };
+}
+
+function equalAnyDistanceTimes(
+  distance: number,
+  times: number,
+  strict: boolean,
+): Feature {
+  return {
+    name: strict
+      ? `has equal letters with ${distance.toString()} letters between, ${times.toString()} times`
+      : `has equal letters with ${distance.toString()} letters between, at least ${times.toString()} times`,
+    property: (slug) => {
+      const starts = interval(0, slug.length - distance - 1).filter((i) => {
+        return slug[i] === slug[i + distance + 1];
+      });
+      if (strict ? starts.length !== times : starts.length < times) {
+        return null;
+      }
+      return printIndexSlug(
+        slug,
+        starts.flatMap((i) => [i, i + distance + 1]),
+      );
+    },
+  };
+}
+
+function bigramOfTimes(
+  bigram: { type: string; check: (a: string, b: string) => boolean },
+  times: number,
+  strict: boolean,
+): Feature {
+  return {
+    name: strict
+      ? `has ${times.toString()} ${bigram.type} bigrams`
+      : `has at least ${times.toString()} ${bigram.type} bigrams`,
+    property: (slug) => {
+      const starts = interval(0, slug.length - 2).filter((i) => {
+        return bigram.check(slug[i]!, slug[i + 1]!);
+      });
+      if (strict ? starts.length !== times : starts.length < times) {
+        return null;
+      }
+      return printIndexSlug(
+        slug,
+        starts.flatMap((i) => [i, i + 1]),
+      );
+    },
+  };
+}
+
+const bigram = {
+  alpha: {
+    type: "alphabetical",
+    check: (a, b) => a < b,
+  },
+  revAlpha: {
+    type: "reverse alphabetical",
+    check: (a, b) => a > b,
+  },
+  seq: {
+    type: "sequential",
+    check: (a, b) => a.charCodeAt(0) - b.charCodeAt(0) === -1,
+  },
+  revSeq: {
+    type: "reverse sequential",
+    check: (a, b) => a.charCodeAt(0) - b.charCodeAt(0) === 1,
+  },
+} as const satisfies Record<
+  string,
+  { type: string; check: (a: string, b: string) => boolean }
+>;
+
+function consecutiveOfTimes(
+  kind: { name: string; letters: string },
+  times: number,
+  strict: boolean,
+): Feature {
+  return {
+    name: strict
+      ? `has ${times.toString()} ${kind.name} in a row`
+      : `has at least ${times.toString()} ${kind.name} in a row`,
+    property: (slug) => {
+      let bestStreak = 0;
+      let bestStart = -1;
+      let currentStreak = 0;
+      for (const [i, letter] of enumerate(slug)) {
+        if (kind.letters.includes(letter)) {
+          currentStreak++;
+          if (currentStreak > bestStreak) {
+            bestStreak = currentStreak;
+            bestStart = i - currentStreak + 1;
+          }
+        } else {
+          currentStreak = 0;
+        }
+      }
+      if (strict ? bestStreak !== times : bestStreak < times) {
+        return null;
+      }
+      return printIndexSlug(
+        slug,
+        interval(bestStart, bestStart + bestStreak - 1),
+      );
+    },
+  };
+}
+
+// TODO: does this belong somewhere else?
+function bookendsOf(length: number): Feature {
+  return {
+    name: `starts and ends with the same ${length.toString()} letters`,
+    property: (slug) => {
+      if (slug.length < length * 2) {
+        return null;
+      }
+      if (slug.slice(0, length) !== slug.slice(-length)) {
+        return null;
+      }
+      return printIndexSlug(slug, [
+        ...interval(0, length - 1),
+        ...interval(slug.length - length, slug.length - 1),
+      ]);
+    },
+  };
+}
+
+/**
+ * Features for letter sequences: things we can remark based on the relative
+ * order of the letters/bigrams/trigrams within the slug.
+ */
+export function letterSequenceFeatures(): Feature[] {
+  return [
+    ...mapProduct(equalWithDistanceTimes, LETTERS, [0, 1, 2, 3], [1]),
+    ...mapProduct(equalWithDistanceTimes, LETTERS, [0, 1], [2, 3]),
+    ...mapProduct(unequalWithDistance, LETTERS, LETTERS, [0, 1]),
+    ...mapProduct(
+      equalAnyDistanceTimes,
+      [0, 1, 2, 3],
+      [1, 2, 3],
+      [true, false],
+    ),
+    ...mapProduct(
+      bigramOfTimes,
+      [bigram.alpha, bigram.revAlpha],
+      interval(1, 10),
+      [true, false],
+    ),
+    ...mapProduct(bigramOfTimes, [bigram.seq, bigram.revSeq], interval(1, 5), [
+      true,
+      false,
+    ]),
+    ...mapProduct(
+      bigramOfTimes,
+      [bigram.alpha, bigram.revAlpha, bigram.seq, bigram.revSeq],
+      [0],
+      [true],
+    ),
+    ...mapProduct(
+      consecutiveOfTimes,
+      [
+        { name: "vowels", letters: VOWELS },
+        { name: "consonants", letters: CONSONANTS },
+      ],
+      interval(2, 5),
+      [true, false],
+    ),
+    ...mapProduct(bookendsOf, [1, 2, 3]),
+  ];
+}

package/src/features/logProbCache.ts

@@ -0,0 +1,48 @@
+import { knownLogProbs } from "../data/knownLogProbs.js";
+import { LogNum } from "../lib/logNum.js";
+
+class LogProbCache {
+  useCache: boolean;
+  knownLogProbs: Record<string, LogNum>;
+  wrapCompute: (
+    name: string,
+    fn: () => LogNum,
+    existing: LogNum | undefined,
+  ) => LogNum;
+
+  constructor(data: Record<string, number>) {
+    this.useCache = true;
+    this.knownLogProbs = {};
+    for (const [name, logProb] of Object.entries(data)) {
+      this.knownLogProbs[name] = LogNum.fromExp(logProb);
+    }
+    this.wrapCompute = (_, fn) => fn();
+  }
+
+  *dump(): Generator<string> {
+    yield `export const knownLogProbs: Record<string, number> = {`;
+    for (const [name, logProb] of Object.entries(this.knownLogProbs)) {
+      yield `  "${name}": ${logProb.toLog().toString()},`;
+    }
+    yield `};`;
+  }
+
+  get(name: string, defaultLogProb: () => LogNum): LogNum {
+    if (this.useCache && name in this.knownLogProbs) {
+      return this.knownLogProbs[name]!;
+    }
+    const logProb = this.wrapCompute(
+      name,
+      defaultLogProb,
+      this.knownLogProbs[name],
+    );
+    this.knownLogProbs[name] = logProb;
+    return logProb;
+  }
+}
+
+/**
+ * We wrap the log prob cache in a class so we can do stuff like print
+ * debug output and whatever.
+ */
+export const KnownLogProbs = new LogProbCache(knownLogProbs);