wiki-graph-core 0.3.1

package/dist/wikipage/cache.js

@@ -0,0 +1,404 @@
+import { stat } from "fs/promises";
+import { resolveWikiGraphCacheDatabasePath } from "../common/wiki-graph-dir.js";
+import { getNumber } from "../document/database.js";
+import { openSharedStateDatabase } from "../document/index.js";
+import { isNodeError } from "../utils/node-error.js";
+const CREATE_QID_CACHE_SQL = `
+CREATE TABLE IF NOT EXISTS qid_cache (
+  qid TEXT NOT NULL,
+  language TEXT NOT NULL,
+  label TEXT,
+  description TEXT,
+  pages_json TEXT NOT NULL,
+  checked_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  PRIMARY KEY (qid, language)
+);
+`;
+const CREATE_DISAMBIGUATION_CACHE_SQL = `
+CREATE TABLE IF NOT EXISTS disambiguation_cache (
+  qid TEXT NOT NULL,
+  wiki TEXT NOT NULL,
+  pages_json TEXT NOT NULL,
+  profile_json TEXT,
+  checked_at TEXT NOT NULL,
+  PRIMARY KEY (qid, wiki)
+);
+`;
+const WIKIPAGE_CACHE_SCHEMA_SQL = `
+${CREATE_QID_CACHE_SQL}
+
+CREATE INDEX IF NOT EXISTS idx_qid_cache_checked_at
+ON qid_cache(checked_at);
+
+${CREATE_DISAMBIGUATION_CACHE_SQL}
+
+CREATE INDEX IF NOT EXISTS idx_disambiguation_cache_checked_at
+ON disambiguation_cache(checked_at);
+`;
+const WIKIPAGE_CACHE_TTL_MS = 30 * 24 * 60 * 60 * 1e3;
+class WikipageCache {
+  #database;
+  constructor(database) {
+    this.#database = database;
+  }
+  static async open(path) {
+    const databasePath = path ?? resolveWikiGraphCacheDatabasePath();
+    const database = await openSharedStateDatabase(
+      databasePath,
+      WIKIPAGE_CACHE_SCHEMA_SQL
+    );
+    try {
+      await migrateWikipageCacheSchema(database);
+    } catch (error) {
+      await database.close().catch(() => void 0);
+      throw error;
+    }
+    return new WikipageCache(database);
+  }
+  async close() {
+    await this.#database.close();
+  }
+  async getQids(qids, language) {
+    if (qids.length === 0) {
+      return /* @__PURE__ */ new Map();
+    }
+    const results = /* @__PURE__ */ new Map();
+    for (const qid of qids) {
+      const record = await this.#database.queryOne(
+        `
+SELECT *
+FROM qid_cache
+WHERE qid = ? AND language = ?
+`,
+        [qid, language],
+        mapQidRecord
+      );
+      if (record !== void 0) {
+        results.set(qid, record);
+      }
+    }
+    return results;
+  }
+  async putQids(records, language) {
+    if (records.length === 0) {
+      return;
+    }
+    await this.#database.transaction(async () => {
+      for (const record of records) {
+        await this.#database.run(
+          `
+INSERT INTO qid_cache (
+  qid, language, label, description, pages_json, checked_at, updated_at
+) VALUES (?, ?, ?, ?, ?, ?, ?)
+ON CONFLICT(qid, language) DO UPDATE SET
+  label = excluded.label,
+  description = excluded.description,
+  pages_json = excluded.pages_json,
+  checked_at = excluded.checked_at,
+  updated_at = excluded.updated_at
+`,
+          [
+            record.qid,
+            language,
+            record.label ?? null,
+            record.description ?? null,
+            JSON.stringify(record.sitelinks),
+            record.checkedAt,
+            record.updatedAt
+          ]
+        );
+      }
+    });
+  }
+  async getDisambiguations(qids, wiki) {
+    if (qids.length === 0) {
+      return /* @__PURE__ */ new Map();
+    }
+    const results = /* @__PURE__ */ new Map();
+    for (const qid of qids) {
+      const record = await this.#database.queryOne(
+        `
+SELECT *
+FROM disambiguation_cache
+WHERE qid = ? AND wiki = ?
+`,
+        [qid, wiki],
+        mapDisambiguationRecord
+      );
+      if (record !== void 0) {
+        results.set(qid, record);
+      }
+    }
+    return results;
+  }
+  async putDisambiguations(records, wiki) {
+    if (records.length === 0) {
+      return;
+    }
+    await this.#database.transaction(async () => {
+      for (const record of records) {
+        await this.#database.run(
+          `
+INSERT INTO disambiguation_cache (
+  qid, wiki, pages_json, profile_json, checked_at
+) VALUES (?, ?, ?, ?, ?)
+ON CONFLICT(qid, wiki) DO UPDATE SET
+  pages_json = excluded.pages_json,
+  profile_json = excluded.profile_json,
+  checked_at = excluded.checked_at
+`,
+          [
+            record.disambiguationQid,
+            wiki,
+            JSON.stringify(record.pages),
+            record.profile === void 0 ? null : JSON.stringify(record.profile),
+            record.checkedAt
+          ]
+        );
+      }
+    });
+  }
+}
+async function runWikipageCacheGc(context) {
+  const databasePath = resolveWikiGraphCacheDatabasePath();
+  const beforeBytes = await readFileSize(databasePath);
+  if (beforeBytes === void 0) {
+    return {
+      freedBytes: 0,
+      removed: 0,
+      scanned: 0
+    };
+  }
+  const database = await openSharedStateDatabase(
+    databasePath,
+    WIKIPAGE_CACHE_SCHEMA_SQL
+  );
+  try {
+    await migrateWikipageCacheSchema(database);
+    const scanned = await countWikipageCacheRows(database);
+    const cutoff = new Date(context.now - WIKIPAGE_CACHE_TTL_MS).toISOString();
+    const expired = await countExpiredWikipageCacheRows(database, cutoff);
+    if (!context.dryRun && expired > 0) {
+      await database.transaction(async () => {
+        await database.run("DELETE FROM qid_cache WHERE checked_at < ?", [
+          cutoff
+        ]);
+        await database.run(
+          "DELETE FROM disambiguation_cache WHERE checked_at < ?",
+          [cutoff]
+        );
+      });
+      await database.run("VACUUM");
+    }
+    const afterBytes = await readFileSize(databasePath);
+    return {
+      freedBytes: context.dryRun ? estimateDryRunFreedBytes(beforeBytes, scanned, expired) : Math.max(0, beforeBytes - (afterBytes ?? 0)),
+      removed: expired,
+      scanned
+    };
+  } finally {
+    await database.close();
+  }
+}
+function estimateDryRunFreedBytes(beforeBytes, scanned, expired) {
+  if (scanned === 0 || expired === 0) {
+    return 0;
+  }
+  return Math.round(beforeBytes * (expired / scanned));
+}
+async function migrateWikipageCacheSchema(database) {
+  await migrateQidCacheSchema(database);
+  await migrateDisambiguationCacheSchema(database);
+}
+async function migrateQidCacheSchema(database) {
+  const columns = await listTableColumns(database, "qid_cache");
+  if (columns.has("language")) {
+    return;
+  }
+  await database.transaction(async () => {
+    const transactionColumns = await listTableColumns(database, "qid_cache");
+    if (transactionColumns.has("language")) {
+      return;
+    }
+    await database.run("ALTER TABLE qid_cache RENAME TO qid_cache_legacy");
+    await database.run(CREATE_QID_CACHE_SQL);
+    await database.run(`
+INSERT OR IGNORE INTO qid_cache (
+  qid, language, label, description, pages_json, checked_at, updated_at
+)
+SELECT qid, 'en', label, description, pages_json, checked_at, updated_at
+FROM qid_cache_legacy
+`);
+    await database.run("DROP TABLE qid_cache_legacy");
+  });
+}
+async function migrateDisambiguationCacheSchema(database) {
+  const columns = await listTableColumns(database, "disambiguation_cache");
+  if (columns.has("wiki")) {
+    return;
+  }
+  await database.transaction(async () => {
+    const transactionColumns = await listTableColumns(
+      database,
+      "disambiguation_cache"
+    );
+    if (transactionColumns.has("wiki")) {
+      return;
+    }
+    await database.run(
+      "ALTER TABLE disambiguation_cache RENAME TO disambiguation_cache_legacy"
+    );
+    await database.run(CREATE_DISAMBIGUATION_CACHE_SQL);
+    await database.run(`
+INSERT OR IGNORE INTO disambiguation_cache (
+  qid, wiki, pages_json, profile_json, checked_at
+)
+SELECT qid, 'enwiki', pages_json, profile_json, checked_at
+FROM disambiguation_cache_legacy
+`);
+    await database.run("DROP TABLE disambiguation_cache_legacy");
+  });
+}
+async function listTableColumns(database, table) {
+  const columns = await database.queryAll(
+    `PRAGMA table_info(${table})`,
+    void 0,
+    (row) => getString(row.name, "name")
+  );
+  return new Set(columns);
+}
+async function countWikipageCacheRows(database) {
+  const qids = await countRows(database, "qid_cache");
+  const disambiguations = await countRows(database, "disambiguation_cache");
+  return qids + disambiguations;
+}
+async function countExpiredWikipageCacheRows(database, cutoff) {
+  const qids = await countRows(database, "qid_cache", cutoff);
+  const disambiguations = await countRows(
+    database,
+    "disambiguation_cache",
+    cutoff
+  );
+  return qids + disambiguations;
+}
+async function countRows(database, table, checkedBefore) {
+  return await database.queryOne(
+    `
+SELECT COUNT(*) AS count
+FROM ${table}
+${checkedBefore === void 0 ? "" : "WHERE checked_at < ?"}
+`,
+    checkedBefore === void 0 ? void 0 : [checkedBefore],
+    (row) => getNumber(row, "count")
+  ) ?? 0;
+}
+async function readFileSize(path) {
+  try {
+    return (await stat(path)).size;
+  } catch (error) {
+    if (isNodeError(error) && error.code === "ENOENT") {
+      return void 0;
+    }
+    throw error;
+  }
+}
+function mapQidRecord(row) {
+  const description = getOptionalString(row.description);
+  const label = getOptionalString(row.label);
+  return {
+    checkedAt: getString(row.checked_at, "checked_at"),
+    ...description === void 0 ? {} : { description },
+    ...label === void 0 ? {} : { label },
+    qid: getString(row.qid, "qid"),
+    sitelinks: parsePageRecords(getString(row.pages_json, "pages_json")),
+    updatedAt: getString(row.updated_at, "updated_at")
+  };
+}
+function mapDisambiguationRecord(row) {
+  return {
+    checkedAt: getString(row.checked_at, "checked_at"),
+    disambiguationQid: getString(row.qid, "qid"),
+    pages: parseDisambiguationPages(getString(row.pages_json, "pages_json")),
+    ...getOptionalString(row.profile_json) === void 0 ? {} : {
+      profile: parseDisambiguationProfile(
+        getOptionalString(row.profile_json)
+      )
+    }
+  };
+}
+function parsePageRecords(value) {
+  const parsed = JSON.parse(value);
+  if (!Array.isArray(parsed)) {
+    return [];
+  }
+  return parsed.filter(isCachedPageRecord);
+}
+function parseDisambiguationPages(value) {
+  const parsed = JSON.parse(value);
+  if (!Array.isArray(parsed)) {
+    return [];
+  }
+  return parsed.filter(isDisambiguationPageText);
+}
+function isCachedPageRecord(value) {
+  if (typeof value !== "object" || value === null) {
+    return false;
+  }
+  const record = value;
+  return typeof record.isDisambiguation === "boolean" && typeof record.title === "string" && record.title !== "" && isSupportedWiki(record.wiki);
+}
+function isDisambiguationPageText(value) {
+  if (typeof value !== "object" || value === null) {
+    return false;
+  }
+  const record = value;
+  return Array.isArray(record.linkedQids) && record.linkedQids.every(isLinkedQid) && typeof record.text === "string" && record.text !== "" && typeof record.title === "string" && record.title !== "" && isSupportedWiki(record.wiki);
+}
+function parseDisambiguationProfile(value) {
+  const parsed = JSON.parse(value);
+  if (!isDisambiguationProfile(parsed)) {
+    throw new Error(
+      "Expected profile_json to contain a disambiguation profile."
+    );
+  }
+  return parsed;
+}
+function isDisambiguationProfile(value) {
+  if (typeof value !== "object" || value === null) {
+    return false;
+  }
+  const record = value;
+  return Array.isArray(record.meanings) && record.meanings.every(isDisambiguationProfileMeaning) && typeof record.sourceQid === "string" && /^Q[1-9]\d*$/u.test(record.sourceQid) && (!("surface" in record) || typeof record.surface === "string");
+}
+function isDisambiguationProfileMeaning(value) {
+  if (typeof value !== "object" || value === null) {
+    return false;
+  }
+  const record = value;
+  return (!("category" in record) || typeof record.category === "string") && typeof record.information === "string" && typeof record.name === "string" && record.name !== "" && (record.priority === "primary" || record.priority === "secondary" || record.priority === "other") && typeof record.qid === "string" && /^Q[1-9]\d*$/u.test(record.qid);
+}
+function isLinkedQid(value) {
+  if (typeof value !== "object" || value === null) {
+    return false;
+  }
+  const record = value;
+  return typeof record.qid === "string" && /^Q[1-9]\d*$/u.test(record.qid) && typeof record.title === "string" && record.title !== "";
+}
+function isSupportedWiki(value) {
+  return value === "enwiki" || value === "zhwiki";
+}
+function getString(value, field) {
+  if (typeof value !== "string") {
+    throw new Error(`Expected ${field} to be a string.`);
+  }
+  return value;
+}
+function getOptionalString(value) {
+  return typeof value === "string" && value !== "" ? value : void 0;
+}
+export {
+  WikipageCache,
+  runWikipageCacheGc
+};
+//# sourceMappingURL=cache.js.map

package/dist/wikipage/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/wikipage/cache.ts"],"sourcesContent":["import { stat } from \"fs/promises\";\n\nimport { resolveWikiGraphCacheDatabasePath } from \"../common/wiki-graph-dir.js\";\nimport { getNumber } from \"../document/database.js\";\nimport { openSharedStateDatabase } from \"../document/index.js\";\nimport type { Database } from \"../document/index.js\";\nimport type { GcContext, GcJobResult } from \"../gc/index.js\";\nimport { isNodeError } from \"../utils/node-error.js\";\n\nimport type {\n  CachedDisambiguationRecord,\n  CachedPageRecord,\n  CachedQidRecord,\n  DisambiguationLinkedQid,\n  DisambiguationPageText,\n  DisambiguationProfile,\n  DisambiguationProfileMeaning,\n} from \"./types.js\";\n\ntype SqlRow = Record<string, unknown>;\n\nconst CREATE_QID_CACHE_SQL = `\nCREATE TABLE IF NOT EXISTS qid_cache (\n  qid TEXT NOT NULL,\n  language TEXT NOT NULL,\n  label TEXT,\n  description TEXT,\n  pages_json TEXT NOT NULL,\n  checked_at TEXT NOT NULL,\n  updated_at TEXT NOT NULL,\n  PRIMARY KEY (qid, language)\n);\n`;\n\nconst CREATE_DISAMBIGUATION_CACHE_SQL = `\nCREATE TABLE IF NOT EXISTS disambiguation_cache (\n  qid TEXT NOT NULL,\n  wiki TEXT NOT NULL,\n  pages_json TEXT NOT NULL,\n  profile_json TEXT,\n  checked_at TEXT NOT NULL,\n  PRIMARY KEY (qid, wiki)\n);\n`;\n\nconst WIKIPAGE_CACHE_SCHEMA_SQL = `\n${CREATE_QID_CACHE_SQL}\n\nCREATE INDEX IF NOT EXISTS idx_qid_cache_checked_at\nON qid_cache(checked_at);\n\n${CREATE_DISAMBIGUATION_CACHE_SQL}\n\nCREATE INDEX IF NOT EXISTS idx_disambiguation_cache_checked_at\nON disambiguation_cache(checked_at);\n`;\nconst WIKIPAGE_CACHE_TTL_MS = 30 * 24 * 60 * 60 * 1000;\n\nexport class WikipageCache {\n  readonly #database: Database;\n\n  public constructor(database: Database) {\n    this.#database = database;\n  }\n\n  public static async open(path?: string): Promise<WikipageCache> {\n    const databasePath = path ?? resolveWikiGraphCacheDatabasePath();\n    const database = await openSharedStateDatabase(\n      databasePath,\n      WIKIPAGE_CACHE_SCHEMA_SQL,\n    );\n\n    try {\n      await migrateWikipageCacheSchema(database);\n    } catch (error) {\n      await database.close().catch(() => undefined);\n      throw error;\n    }\n\n    return new WikipageCache(database);\n  }\n\n  public async close(): Promise<void> {\n    await this.#database.close();\n  }\n\n  public async getQids(\n    qids: readonly string[],\n    language: string,\n  ): Promise<ReadonlyMap<string, CachedQidRecord>> {\n    if (qids.length === 0) {\n      return new Map();\n    }\n\n    const results = new Map<string, CachedQidRecord>();\n\n    for (const qid of qids) {\n      const record = await this.#database.queryOne(\n        `\nSELECT *\nFROM qid_cache\nWHERE qid = ? AND language = ?\n`,\n        [qid, language],\n        mapQidRecord,\n      );\n\n      if (record !== undefined) {\n        results.set(qid, record);\n      }\n    }\n\n    return results;\n  }\n\n  public async putQids(\n    records: readonly CachedQidRecord[],\n    language: string,\n  ): Promise<void> {\n    if (records.length === 0) {\n      return;\n    }\n\n    await this.#database.transaction(async () => {\n      for (const record of records) {\n        await this.#database.run(\n          `\nINSERT INTO qid_cache (\n  qid, language, label, description, pages_json, checked_at, updated_at\n) VALUES (?, ?, ?, ?, ?, ?, ?)\nON CONFLICT(qid, language) DO UPDATE SET\n  label = excluded.label,\n  description = excluded.description,\n  pages_json = excluded.pages_json,\n  checked_at = excluded.checked_at,\n  updated_at = excluded.updated_at\n`,\n          [\n            record.qid,\n            language,\n            record.label ?? null,\n            record.description ?? null,\n            JSON.stringify(record.sitelinks),\n            record.checkedAt,\n            record.updatedAt,\n          ],\n        );\n      }\n    });\n  }\n\n  public async getDisambiguations(\n    qids: readonly string[],\n    wiki: string,\n  ): Promise<ReadonlyMap<string, CachedDisambiguationRecord>> {\n    if (qids.length === 0) {\n      return new Map();\n    }\n\n    const results = new Map<string, CachedDisambiguationRecord>();\n\n    for (const qid of qids) {\n      const record = await this.#database.queryOne(\n        `\nSELECT *\nFROM disambiguation_cache\nWHERE qid = ? AND wiki = ?\n`,\n        [qid, wiki],\n        mapDisambiguationRecord,\n      );\n\n      if (record !== undefined) {\n        results.set(qid, record);\n      }\n    }\n\n    return results;\n  }\n\n  public async putDisambiguations(\n    records: readonly CachedDisambiguationRecord[],\n    wiki: string,\n  ): Promise<void> {\n    if (records.length === 0) {\n      return;\n    }\n\n    await this.#database.transaction(async () => {\n      for (const record of records) {\n        await this.#database.run(\n          `\nINSERT INTO disambiguation_cache (\n  qid, wiki, pages_json, profile_json, checked_at\n) VALUES (?, ?, ?, ?, ?)\nON CONFLICT(qid, wiki) DO UPDATE SET\n  pages_json = excluded.pages_json,\n  profile_json = excluded.profile_json,\n  checked_at = excluded.checked_at\n`,\n          [\n            record.disambiguationQid,\n            wiki,\n            JSON.stringify(record.pages),\n            record.profile === undefined\n              ? null\n              : JSON.stringify(record.profile),\n            record.checkedAt,\n          ],\n        );\n      }\n    });\n  }\n}\n\nexport async function runWikipageCacheGc(\n  context: GcContext,\n): Promise<GcJobResult> {\n  const databasePath = resolveWikiGraphCacheDatabasePath();\n  const beforeBytes = await readFileSize(databasePath);\n\n  if (beforeBytes === undefined) {\n    return {\n      freedBytes: 0,\n      removed: 0,\n      scanned: 0,\n    };\n  }\n\n  const database = await openSharedStateDatabase(\n    databasePath,\n    WIKIPAGE_CACHE_SCHEMA_SQL,\n  );\n\n  try {\n    await migrateWikipageCacheSchema(database);\n\n    const scanned = await countWikipageCacheRows(database);\n    const cutoff = new Date(context.now - WIKIPAGE_CACHE_TTL_MS).toISOString();\n    const expired = await countExpiredWikipageCacheRows(database, cutoff);\n\n    if (!context.dryRun && expired > 0) {\n      await database.transaction(async () => {\n        await database.run(\"DELETE FROM qid_cache WHERE checked_at < ?\", [\n          cutoff,\n        ]);\n        await database.run(\n          \"DELETE FROM disambiguation_cache WHERE checked_at < ?\",\n          [cutoff],\n        );\n      });\n      await database.run(\"VACUUM\");\n    }\n\n    const afterBytes = await readFileSize(databasePath);\n\n    return {\n      freedBytes: context.dryRun\n        ? estimateDryRunFreedBytes(beforeBytes, scanned, expired)\n        : Math.max(0, beforeBytes - (afterBytes ?? 0)),\n      removed: expired,\n      scanned,\n    };\n  } finally {\n    await database.close();\n  }\n}\n\nfunction estimateDryRunFreedBytes(\n  beforeBytes: number,\n  scanned: number,\n  expired: number,\n): number {\n  if (scanned === 0 || expired === 0) {\n    return 0;\n  }\n\n  return Math.round(beforeBytes * (expired / scanned));\n}\n\nasync function migrateWikipageCacheSchema(database: Database): Promise<void> {\n  await migrateQidCacheSchema(database);\n  await migrateDisambiguationCacheSchema(database);\n}\n\nasync function migrateQidCacheSchema(database: Database): Promise<void> {\n  const columns = await listTableColumns(database, \"qid_cache\");\n\n  if (columns.has(\"language\")) {\n    return;\n  }\n\n  await database.transaction(async () => {\n    const transactionColumns = await listTableColumns(database, \"qid_cache\");\n\n    if (transactionColumns.has(\"language\")) {\n      return;\n    }\n\n    await database.run(\"ALTER TABLE qid_cache RENAME TO qid_cache_legacy\");\n    await database.run(CREATE_QID_CACHE_SQL);\n    await database.run(`\nINSERT OR IGNORE INTO qid_cache (\n  qid, language, label, description, pages_json, checked_at, updated_at\n)\nSELECT qid, 'en', label, description, pages_json, checked_at, updated_at\nFROM qid_cache_legacy\n`);\n    await database.run(\"DROP TABLE qid_cache_legacy\");\n  });\n}\n\nasync function migrateDisambiguationCacheSchema(\n  database: Database,\n): Promise<void> {\n  const columns = await listTableColumns(database, \"disambiguation_cache\");\n\n  if (columns.has(\"wiki\")) {\n    return;\n  }\n\n  await database.transaction(async () => {\n    const transactionColumns = await listTableColumns(\n      database,\n      \"disambiguation_cache\",\n    );\n\n    if (transactionColumns.has(\"wiki\")) {\n      return;\n    }\n\n    await database.run(\n      \"ALTER TABLE disambiguation_cache RENAME TO disambiguation_cache_legacy\",\n    );\n    await database.run(CREATE_DISAMBIGUATION_CACHE_SQL);\n    await database.run(`\nINSERT OR IGNORE INTO disambiguation_cache (\n  qid, wiki, pages_json, profile_json, checked_at\n)\nSELECT qid, 'enwiki', pages_json, profile_json, checked_at\nFROM disambiguation_cache_legacy\n`);\n    await database.run(\"DROP TABLE disambiguation_cache_legacy\");\n  });\n}\n\nasync function listTableColumns(\n  database: Database,\n  table: string,\n): Promise<ReadonlySet<string>> {\n  const columns = await database.queryAll(\n    `PRAGMA table_info(${table})`,\n    undefined,\n    (row) => getString(row.name, \"name\"),\n  );\n\n  return new Set(columns);\n}\n\nasync function countWikipageCacheRows(database: Database): Promise<number> {\n  const qids = await countRows(database, \"qid_cache\");\n  const disambiguations = await countRows(database, \"disambiguation_cache\");\n\n  return qids + disambiguations;\n}\n\nasync function countExpiredWikipageCacheRows(\n  database: Database,\n  cutoff: string,\n): Promise<number> {\n  const qids = await countRows(database, \"qid_cache\", cutoff);\n  const disambiguations = await countRows(\n    database,\n    \"disambiguation_cache\",\n    cutoff,\n  );\n\n  return qids + disambiguations;\n}\n\nasync function countRows(\n  database: Database,\n  table: \"disambiguation_cache\" | \"qid_cache\",\n  checkedBefore?: string,\n): Promise<number> {\n  return (\n    (await database.queryOne(\n      `\nSELECT COUNT(*) AS count\nFROM ${table}\n${checkedBefore === undefined ? \"\" : \"WHERE checked_at < ?\"}\n`,\n      checkedBefore === undefined ? undefined : [checkedBefore],\n      (row) => getNumber(row, \"count\"),\n    )) ?? 0\n  );\n}\n\nasync function readFileSize(path: string): Promise<number | undefined> {\n  try {\n    return (await stat(path)).size;\n  } catch (error) {\n    if (isNodeError(error) && error.code === \"ENOENT\") {\n      return undefined;\n    }\n\n    throw error;\n  }\n}\n\nfunction mapQidRecord(row: SqlRow): CachedQidRecord {\n  const description = getOptionalString(row.description);\n  const label = getOptionalString(row.label);\n\n  return {\n    checkedAt: getString(row.checked_at, \"checked_at\"),\n    ...(description === undefined ? {} : { description }),\n    ...(label === undefined ? {} : { label }),\n    qid: getString(row.qid, \"qid\"),\n    sitelinks: parsePageRecords(getString(row.pages_json, \"pages_json\")),\n    updatedAt: getString(row.updated_at, \"updated_at\"),\n  };\n}\n\nfunction mapDisambiguationRecord(row: SqlRow): CachedDisambiguationRecord {\n  return {\n    checkedAt: getString(row.checked_at, \"checked_at\"),\n    disambiguationQid: getString(row.qid, \"qid\"),\n    pages: parseDisambiguationPages(getString(row.pages_json, \"pages_json\")),\n    ...(getOptionalString(row.profile_json) === undefined\n      ? {}\n      : {\n          profile: parseDisambiguationProfile(\n            getOptionalString(row.profile_json)!,\n          ),\n        }),\n  };\n}\n\nfunction parsePageRecords(value: string): readonly CachedPageRecord[] {\n  const parsed: unknown = JSON.parse(value);\n\n  if (!Array.isArray(parsed)) {\n    return [];\n  }\n\n  return parsed.filter(isCachedPageRecord);\n}\n\nfunction parseDisambiguationPages(\n  value: string,\n): readonly DisambiguationPageText[] {\n  const parsed: unknown = JSON.parse(value);\n\n  if (!Array.isArray(parsed)) {\n    return [];\n  }\n\n  return parsed.filter(isDisambiguationPageText);\n}\n\nfunction isCachedPageRecord(value: unknown): value is CachedPageRecord {\n  if (typeof value !== \"object\" || value === null) {\n    return false;\n  }\n\n  const record = value as Record<string, unknown>;\n\n  return (\n    typeof record.isDisambiguation === \"boolean\" &&\n    typeof record.title === \"string\" &&\n    record.title !== \"\" &&\n    isSupportedWiki(record.wiki)\n  );\n}\n\nfunction isDisambiguationPageText(\n  value: unknown,\n): value is DisambiguationPageText {\n  if (typeof value !== \"object\" || value === null) {\n    return false;\n  }\n\n  const record = value as Record<string, unknown>;\n\n  return (\n    Array.isArray(record.linkedQids) &&\n    record.linkedQids.every(isLinkedQid) &&\n    typeof record.text === \"string\" &&\n    record.text !== \"\" &&\n    typeof record.title === \"string\" &&\n    record.title !== \"\" &&\n    isSupportedWiki(record.wiki)\n  );\n}\n\nfunction parseDisambiguationProfile(value: string): DisambiguationProfile {\n  const parsed: unknown = JSON.parse(value);\n\n  if (!isDisambiguationProfile(parsed)) {\n    throw new Error(\n      \"Expected profile_json to contain a disambiguation profile.\",\n    );\n  }\n\n  return parsed;\n}\n\nfunction isDisambiguationProfile(\n  value: unknown,\n): value is DisambiguationProfile {\n  if (typeof value !== \"object\" || value === null) {\n    return false;\n  }\n\n  const record = value as Record<string, unknown>;\n\n  return (\n    Array.isArray(record.meanings) &&\n    record.meanings.every(isDisambiguationProfileMeaning) &&\n    typeof record.sourceQid === \"string\" &&\n    /^Q[1-9]\\d*$/u.test(record.sourceQid) &&\n    (!(\"surface\" in record) || typeof record.surface === \"string\")\n  );\n}\n\nfunction isDisambiguationProfileMeaning(\n  value: unknown,\n): value is DisambiguationProfileMeaning {\n  if (typeof value !== \"object\" || value === null) {\n    return false;\n  }\n\n  const record = value as Record<string, unknown>;\n\n  return (\n    (!(\"category\" in record) || typeof record.category === \"string\") &&\n    typeof record.information === \"string\" &&\n    typeof record.name === \"string\" &&\n    record.name !== \"\" &&\n    (record.priority === \"primary\" ||\n      record.priority === \"secondary\" ||\n      record.priority === \"other\") &&\n    typeof record.qid === \"string\" &&\n    /^Q[1-9]\\d*$/u.test(record.qid)\n  );\n}\n\nfunction isLinkedQid(value: unknown): value is DisambiguationLinkedQid {\n  if (typeof value !== \"object\" || value === null) {\n    return false;\n  }\n\n  const record = value as Record<string, unknown>;\n\n  return (\n    typeof record.qid === \"string\" &&\n    /^Q[1-9]\\d*$/u.test(record.qid) &&\n    typeof record.title === \"string\" &&\n    record.title !== \"\"\n  );\n}\n\nfunction isSupportedWiki(value: unknown): value is \"enwiki\" | \"zhwiki\" {\n  return value === \"enwiki\" || value === \"zhwiki\";\n}\n\nfunction getString(value: unknown, field: string): string {\n  if (typeof value !== \"string\") {\n    throw new Error(`Expected ${field} to be a string.`);\n  }\n\n  return value;\n}\n\nfunction getOptionalString(value: unknown): string | undefined {\n  return typeof value === \"string\" && value !== \"\" ? value : undefined;\n}\n"],"mappings":"AAAA,SAAS,YAAY;AAErB,SAAS,yCAAyC;AAClD,SAAS,iBAAiB;AAC1B,SAAS,+BAA+B;AAGxC,SAAS,mBAAmB;AAc5B,MAAM,uBAAuB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAa7B,MAAM,kCAAkC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWxC,MAAM,4BAA4B;AAAA,EAChC,oBAAoB;AAAA;AAAA;AAAA;AAAA;AAAA,EAKpB,+BAA+B;AAAA;AAAA;AAAA;AAAA;AAKjC,MAAM,wBAAwB,KAAK,KAAK,KAAK,KAAK;AAE3C,MAAM,cAAc;AAAA,EAChB;AAAA,EAEF,YAAY,UAAoB;AACrC,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,aAAoB,KAAK,MAAuC;AAC9D,UAAM,eAAe,QAAQ,kCAAkC;AAC/D,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,IACF;AAEA,QAAI;AACF,YAAM,2BAA2B,QAAQ;AAAA,IAC3C,SAAS,OAAO;AACd,YAAM,SAAS,MAAM,EAAE,MAAM,MAAM,MAAS;AAC5C,YAAM;AAAA,IACR;AAEA,WAAO,IAAI,cAAc,QAAQ;AAAA,EACnC;AAAA,EAEA,MAAa,QAAuB;AAClC,UAAM,KAAK,UAAU,MAAM;AAAA,EAC7B;AAAA,EAEA,MAAa,QACX,MACA,UAC+C;AAC/C,QAAI,KAAK,WAAW,GAAG;AACrB,aAAO,oBAAI,IAAI;AAAA,IACjB;AAEA,UAAM,UAAU,oBAAI,IAA6B;AAEjD,eAAW,OAAO,MAAM;AACtB,YAAM,SAAS,MAAM,KAAK,UAAU;AAAA,QAClC;AAAA;AAAA;AAAA;AAAA;AAAA,QAKA,CAAC,KAAK,QAAQ;AAAA,QACd;AAAA,MACF;AAEA,UAAI,WAAW,QAAW;AACxB,gBAAQ,IAAI,KAAK,MAAM;AAAA,MACzB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAa,QACX,SACA,UACe;AACf,QAAI,QAAQ,WAAW,GAAG;AACxB;AAAA,IACF;AAEA,UAAM,KAAK,UAAU,YAAY,YAAY;AAC3C,iBAAW,UAAU,SAAS;AAC5B,cAAM,KAAK,UAAU;AAAA,UACnB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,UAWA;AAAA,YACE,OAAO;AAAA,YACP;AAAA,YACA,OAAO,SAAS;AAAA,YAChB,OAAO,eAAe;AAAA,YACtB,KAAK,UAAU,OAAO,SAAS;AAAA,YAC/B,OAAO;AAAA,YACP,OAAO;AAAA,UACT;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAa,mBACX,MACA,MAC0D;AAC1D,QAAI,KAAK,WAAW,GAAG;AACrB,aAAO,oBAAI,IAAI;AAAA,IACjB;AAEA,UAAM,UAAU,oBAAI,IAAwC;AAE5D,eAAW,OAAO,MAAM;AACtB,YAAM,SAAS,MAAM,KAAK,UAAU;AAAA,QAClC;AAAA;AAAA;AAAA;AAAA;AAAA,QAKA,CAAC,KAAK,IAAI;AAAA,QACV;AAAA,MACF;AAEA,UAAI,WAAW,QAAW;AACxB,gBAAQ,IAAI,KAAK,MAAM;AAAA,MACzB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAa,mBACX,SACA,MACe;AACf,QAAI,QAAQ,WAAW,GAAG;AACxB;AAAA,IACF;AAEA,UAAM,KAAK,UAAU,YAAY,YAAY;AAC3C,iBAAW,UAAU,SAAS;AAC5B,cAAM,KAAK,UAAU;AAAA,UACnB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,UASA;AAAA,YACE,OAAO;AAAA,YACP;AAAA,YACA,KAAK,UAAU,OAAO,KAAK;AAAA,YAC3B,OAAO,YAAY,SACf,OACA,KAAK,UAAU,OAAO,OAAO;AAAA,YACjC,OAAO;AAAA,UACT;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEA,eAAsB,mBACpB,SACsB;AACtB,QAAM,eAAe,kCAAkC;AACvD,QAAM,cAAc,MAAM,aAAa,YAAY;AAEnD,MAAI,gBAAgB,QAAW;AAC7B,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,SAAS;AAAA,MACT,SAAS;AAAA,IACX;AAAA,EACF;AAEA,QAAM,WAAW,MAAM;AAAA,IACrB;AAAA,IACA;AAAA,EACF;AAEA,MAAI;AACF,UAAM,2BAA2B,QAAQ;AAEzC,UAAM,UAAU,MAAM,uBAAuB,QAAQ;AACrD,UAAM,SAAS,IAAI,KAAK,QAAQ,MAAM,qBAAqB,EAAE,YAAY;AACzE,UAAM,UAAU,MAAM,8BAA8B,UAAU,MAAM;AAEpE,QAAI,CAAC,QAAQ,UAAU,UAAU,GAAG;AAClC,YAAM,SAAS,YAAY,YAAY;AACrC,cAAM,SAAS,IAAI,8CAA8C;AAAA,UAC/D;AAAA,QACF,CAAC;AACD,cAAM,SAAS;AAAA,UACb;AAAA,UACA,CAAC,MAAM;AAAA,QACT;AAAA,MACF,CAAC;AACD,YAAM,SAAS,IAAI,QAAQ;AAAA,IAC7B;AAEA,UAAM,aAAa,MAAM,aAAa,YAAY;AAElD,WAAO;AAAA,MACL,YAAY,QAAQ,SAChB,yBAAyB,aAAa,SAAS,OAAO,IACtD,KAAK,IAAI,GAAG,eAAe,cAAc,EAAE;AAAA,MAC/C,SAAS;AAAA,MACT;AAAA,IACF;AAAA,EACF,UAAE;AACA,UAAM,SAAS,MAAM;AAAA,EACvB;AACF;AAEA,SAAS,yBACP,aACA,SACA,SACQ;AACR,MAAI,YAAY,KAAK,YAAY,GAAG;AAClC,WAAO;AAAA,EACT;AAEA,SAAO,KAAK,MAAM,eAAe,UAAU,QAAQ;AACrD;AAEA,eAAe,2BAA2B,UAAmC;AAC3E,QAAM,sBAAsB,QAAQ;AACpC,QAAM,iCAAiC,QAAQ;AACjD;AAEA,eAAe,sBAAsB,UAAmC;AACtE,QAAM,UAAU,MAAM,iBAAiB,UAAU,WAAW;AAE5D,MAAI,QAAQ,IAAI,UAAU,GAAG;AAC3B;AAAA,EACF;AAEA,QAAM,SAAS,YAAY,YAAY;AACrC,UAAM,qBAAqB,MAAM,iBAAiB,UAAU,WAAW;AAEvE,QAAI,mBAAmB,IAAI,UAAU,GAAG;AACtC;AAAA,IACF;AAEA,UAAM,SAAS,IAAI,kDAAkD;AACrE,UAAM,SAAS,IAAI,oBAAoB;AACvC,UAAM,SAAS,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,CAMtB;AACG,UAAM,SAAS,IAAI,6BAA6B;AAAA,EAClD,CAAC;AACH;AAEA,eAAe,iCACb,UACe;AACf,QAAM,UAAU,MAAM,iBAAiB,UAAU,sBAAsB;AAEvE,MAAI,QAAQ,IAAI,MAAM,GAAG;AACvB;AAAA,EACF;AAEA,QAAM,SAAS,YAAY,YAAY;AACrC,UAAM,qBAAqB,MAAM;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,mBAAmB,IAAI,MAAM,GAAG;AAClC;AAAA,IACF;AAEA,UAAM,SAAS;AAAA,MACb;AAAA,IACF;AACA,UAAM,SAAS,IAAI,+BAA+B;AAClD,UAAM,SAAS,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,CAMtB;AACG,UAAM,SAAS,IAAI,wCAAwC;AAAA,EAC7D,CAAC;AACH;AAEA,eAAe,iBACb,UACA,OAC8B;AAC9B,QAAM,UAAU,MAAM,SAAS;AAAA,IAC7B,qBAAqB,KAAK;AAAA,IAC1B;AAAA,IACA,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM;AAAA,EACrC;AAEA,SAAO,IAAI,IAAI,OAAO;AACxB;AAEA,eAAe,uBAAuB,UAAqC;AACzE,QAAM,OAAO,MAAM,UAAU,UAAU,WAAW;AAClD,QAAM,kBAAkB,MAAM,UAAU,UAAU,sBAAsB;AAExE,SAAO,OAAO;AAChB;AAEA,eAAe,8BACb,UACA,QACiB;AACjB,QAAM,OAAO,MAAM,UAAU,UAAU,aAAa,MAAM;AAC1D,QAAM,kBAAkB,MAAM;AAAA,IAC5B;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,SAAO,OAAO;AAChB;AAEA,eAAe,UACb,UACA,OACA,eACiB;AACjB,SACG,MAAM,SAAS;AAAA,IACd;AAAA;AAAA,OAEC,KAAK;AAAA,EACV,kBAAkB,SAAY,KAAK,sBAAsB;AAAA;AAAA,IAErD,kBAAkB,SAAY,SAAY,CAAC,aAAa;AAAA,IACxD,CAAC,QAAQ,UAAU,KAAK,OAAO;AAAA,EACjC,KAAM;AAEV;AAEA,eAAe,aAAa,MAA2C;AACrE,MAAI;AACF,YAAQ,MAAM,KAAK,IAAI,GAAG;AAAA,EAC5B,SAAS,OAAO;AACd,QAAI,YAAY,KAAK,KAAK,MAAM,SAAS,UAAU;AACjD,aAAO;AAAA,IACT;AAEA,UAAM;AAAA,EACR;AACF;AAEA,SAAS,aAAa,KAA8B;AAClD,QAAM,cAAc,kBAAkB,IAAI,WAAW;AACrD,QAAM,QAAQ,kBAAkB,IAAI,KAAK;AAEzC,SAAO;AAAA,IACL,WAAW,UAAU,IAAI,YAAY,YAAY;AAAA,IACjD,GAAI,gBAAgB,SAAY,CAAC,IAAI,EAAE,YAAY;AAAA,IACnD,GAAI,UAAU,SAAY,CAAC,IAAI,EAAE,MAAM;AAAA,IACvC,KAAK,UAAU,IAAI,KAAK,KAAK;AAAA,IAC7B,WAAW,iBAAiB,UAAU,IAAI,YAAY,YAAY,CAAC;AAAA,IACnE,WAAW,UAAU,IAAI,YAAY,YAAY;AAAA,EACnD;AACF;AAEA,SAAS,wBAAwB,KAAyC;AACxE,SAAO;AAAA,IACL,WAAW,UAAU,IAAI,YAAY,YAAY;AAAA,IACjD,mBAAmB,UAAU,IAAI,KAAK,KAAK;AAAA,IAC3C,OAAO,yBAAyB,UAAU,IAAI,YAAY,YAAY,CAAC;AAAA,IACvE,GAAI,kBAAkB,IAAI,YAAY,MAAM,SACxC,CAAC,IACD;AAAA,MACE,SAAS;AAAA,QACP,kBAAkB,IAAI,YAAY;AAAA,MACpC;AAAA,IACF;AAAA,EACN;AACF;AAEA,SAAS,iBAAiB,OAA4C;AACpE,QAAM,SAAkB,KAAK,MAAM,KAAK;AAExC,MAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,WAAO,CAAC;AAAA,EACV;AAEA,SAAO,OAAO,OAAO,kBAAkB;AACzC;AAEA,SAAS,yBACP,OACmC;AACnC,QAAM,SAAkB,KAAK,MAAM,KAAK;AAExC,MAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,WAAO,CAAC;AAAA,EACV;AAEA,SAAO,OAAO,OAAO,wBAAwB;AAC/C;AAEA,SAAS,mBAAmB,OAA2C;AACrE,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AAEA,QAAM,SAAS;AAEf,SACE,OAAO,OAAO,qBAAqB,aACnC,OAAO,OAAO,UAAU,YACxB,OAAO,UAAU,MACjB,gBAAgB,OAAO,IAAI;AAE/B;AAEA,SAAS,yBACP,OACiC;AACjC,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AAEA,QAAM,SAAS;AAEf,SACE,MAAM,QAAQ,OAAO,UAAU,KAC/B,OAAO,WAAW,MAAM,WAAW,KACnC,OAAO,OAAO,SAAS,YACvB,OAAO,SAAS,MAChB,OAAO,OAAO,UAAU,YACxB,OAAO,UAAU,MACjB,gBAAgB,OAAO,IAAI;AAE/B;AAEA,SAAS,2BAA2B,OAAsC;AACxE,QAAM,SAAkB,KAAK,MAAM,KAAK;AAExC,MAAI,CAAC,wBAAwB,MAAM,GAAG;AACpC,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,wBACP,OACgC;AAChC,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AAEA,QAAM,SAAS;AAEf,SACE,MAAM,QAAQ,OAAO,QAAQ,KAC7B,OAAO,SAAS,MAAM,8BAA8B,KACpD,OAAO,OAAO,cAAc,YAC5B,eAAe,KAAK,OAAO,SAAS,MACnC,EAAE,aAAa,WAAW,OAAO,OAAO,YAAY;AAEzD;AAEA,SAAS,+BACP,OACuC;AACvC,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AAEA,QAAM,SAAS;AAEf,UACG,EAAE,cAAc,WAAW,OAAO,OAAO,aAAa,aACvD,OAAO,OAAO,gBAAgB,YAC9B,OAAO,OAAO,SAAS,YACvB,OAAO,SAAS,OACf,OAAO,aAAa,aACnB,OAAO,aAAa,eACpB,OAAO,aAAa,YACtB,OAAO,OAAO,QAAQ,YACtB,eAAe,KAAK,OAAO,GAAG;AAElC;AAEA,SAAS,YAAY,OAAkD;AACrE,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AAEA,QAAM,SAAS;AAEf,SACE,OAAO,OAAO,QAAQ,YACtB,eAAe,KAAK,OAAO,GAAG,KAC9B,OAAO,OAAO,UAAU,YACxB,OAAO,UAAU;AAErB;AAEA,SAAS,gBAAgB,OAA8C;AACrE,SAAO,UAAU,YAAY,UAAU;AACzC;AAEA,SAAS,UAAU,OAAgB,OAAuB;AACxD,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,IAAI,MAAM,YAAY,KAAK,kBAAkB;AAAA,EACrD;AAEA,SAAO;AACT;AAEA,SAAS,kBAAkB,OAAoC;AAC7D,SAAO,OAAO,UAAU,YAAY,UAAU,KAAK,QAAQ;AAC7D;","names":[]}

package/dist/wikipage/fetch-log.d.ts

@@ -0,0 +1,17 @@
+interface WikipageFetchLog {
+    readonly filePath: string | undefined;
+    append(entry: WikipageFetchLogEntry): Promise<void>;
+    warnFailed(): void;
+}
+interface WikipageFetchLogEntry {
+    readonly attempt: number;
+    readonly durationMs: number;
+    readonly error?: unknown;
+    readonly response?: Response;
+    readonly responseText?: string;
+    readonly startedAt: number;
+    readonly url: URL;
+}
+declare function createWikipageFetchLog(logDirPath?: string): WikipageFetchLog;
+
+export { type WikipageFetchLog, createWikipageFetchLog };

package/dist/wikipage/fetch-log.js

@@ -0,0 +1,143 @@
+import { createHash } from "crypto";
+import { appendFile } from "fs/promises";
+import { getLogger, resolveArtifactPath } from "../common/logging.js";
+import { formatError } from "../utils/node-error.js";
+const MAX_RESPONSE_TEXT_LENGTH = 16384;
+class SilentWikipageFetchLog {
+  filePath = void 0;
+  append() {
+    return Promise.resolve();
+  }
+  warnFailed() {
+    return void 0;
+  }
+}
+class FileWikipageFetchLog {
+  #filePath;
+  #failedWarned = false;
+  constructor(filePath) {
+    this.#filePath = filePath;
+  }
+  get filePath() {
+    return this.#filePath;
+  }
+  async append(entry) {
+    await appendFile(
+      this.#filePath,
+      `${JSON.stringify(formatEntry(entry))}
+`,
+      "utf8"
+    );
+  }
+  warnFailed() {
+    if (this.#failedWarned) {
+      return;
+    }
+    this.#failedWarned = true;
+    getLogger({ component: "wikipage" }).warn(
+      `
+[Wikipage] Failed with fetch log: ${this.#filePath}`
+    );
+  }
+}
+function createWikipageFetchLog(logDirPath) {
+  if (logDirPath === void 0) {
+    return new SilentWikipageFetchLog();
+  }
+  const filePath = resolveArtifactPath({
+    category: "wikipage",
+    fileName: "wikipage-fetch.jsonl",
+    logDirPath
+  });
+  return filePath === void 0 ? new SilentWikipageFetchLog() : new FileWikipageFetchLog(filePath);
+}
+function formatEntry(entry) {
+  return {
+    action: entry.url.searchParams.get("action") ?? null,
+    attempt: entry.attempt,
+    batch: summarizeBatch(entry.url),
+    durationMs: entry.durationMs,
+    endpoint: `${entry.url.origin}${entry.url.pathname}`,
+    host: entry.url.host,
+    ok: entry.response?.ok ?? false,
+    startedAt: new Date(entry.startedAt).toISOString(),
+    ...entry.response === void 0 ? {} : {
+      retryAfter: entry.response.headers.get("retry-after"),
+      status: entry.response.status
+    },
+    ...entry.error === void 0 ? {} : { error: formatLogError(entry.error) },
+    ...entry.responseText === void 0 ? {} : { responseText: truncateResponseText(entry.responseText) }
+  };
+}
+function summarizeBatch(url) {
+  const ids = url.searchParams.get("ids");
+  if (ids !== null) {
+    return summarizeList("qid", ids.split("|"));
+  }
+  const titles = url.searchParams.get("titles");
+  if (titles !== null) {
+    return summarizeList("title", titles.split("|"));
+  }
+  const page = url.searchParams.get("page");
+  if (page !== null) {
+    return summarizeList("page", [page]);
+  }
+  return void 0;
+}
+function summarizeList(kind, values) {
+  const joined = values.join("|");
+  return {
+    count: values.length,
+    hash: createHash("sha256").update(joined).digest("hex").slice(0, 16),
+    kind,
+    sample: values.slice(0, 5)
+  };
+}
+function formatLogError(error) {
+  return {
+    message: formatError(error),
+    name: error instanceof Error ? error.name : typeof error,
+    ...formatErrorCause(error)
+  };
+}
+function formatErrorCause(error) {
+  if (!(error instanceof Error) || error.cause === void 0) {
+    return {};
+  }
+  const cause = error.cause;
+  return {
+    cause: cause instanceof Error ? {
+      message: formatError(cause),
+      name: cause.name,
+      ...formatNodeErrorCode(cause)
+    } : stringifyUnknown(cause)
+  };
+}
+function stringifyUnknown(value) {
+  if (typeof value === "string") {
+    return value;
+  }
+  if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint" || typeof value === "symbol") {
+    return value.toString();
+  }
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return "unserializable error cause";
+  }
+}
+function formatNodeErrorCode(error) {
+  const code = error.code;
+  return typeof code === "string" ? { code } : {};
+}
+function truncateResponseText(text) {
+  if (text.length <= MAX_RESPONSE_TEXT_LENGTH) {
+    return text;
+  }
+  return `${text.slice(0, MAX_RESPONSE_TEXT_LENGTH)}
+[truncated ${text.length - MAX_RESPONSE_TEXT_LENGTH} chars]`;
+}
+export {
+  createWikipageFetchLog
+};
+//# sourceMappingURL=fetch-log.js.map

package/dist/wikipage/fetch-log.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/wikipage/fetch-log.ts"],"sourcesContent":["import { createHash } from \"crypto\";\nimport { appendFile } from \"fs/promises\";\n\nimport { getLogger, resolveArtifactPath } from \"../common/logging.js\";\nimport { formatError } from \"../utils/node-error.js\";\n\nconst MAX_RESPONSE_TEXT_LENGTH = 16_384;\n\nexport interface WikipageFetchLog {\n  readonly filePath: string | undefined;\n  append(entry: WikipageFetchLogEntry): Promise<void>;\n  warnFailed(): void;\n}\n\ninterface WikipageFetchLogEntry {\n  readonly attempt: number;\n  readonly durationMs: number;\n  readonly error?: unknown;\n  readonly response?: Response;\n  readonly responseText?: string;\n  readonly startedAt: number;\n  readonly url: URL;\n}\n\nclass SilentWikipageFetchLog implements WikipageFetchLog {\n  public readonly filePath = undefined;\n\n  public append(): Promise<void> {\n    return Promise.resolve();\n  }\n\n  public warnFailed(): void {\n    return undefined;\n  }\n}\n\nclass FileWikipageFetchLog implements WikipageFetchLog {\n  readonly #filePath: string;\n  #failedWarned = false;\n\n  public constructor(filePath: string) {\n    this.#filePath = filePath;\n  }\n\n  public get filePath(): string {\n    return this.#filePath;\n  }\n\n  public async append(entry: WikipageFetchLogEntry): Promise<void> {\n    await appendFile(\n      this.#filePath,\n      `${JSON.stringify(formatEntry(entry))}\\n`,\n      \"utf8\",\n    );\n  }\n\n  public warnFailed(): void {\n    if (this.#failedWarned) {\n      return;\n    }\n    this.#failedWarned = true;\n\n    getLogger({ component: \"wikipage\" }).warn(\n      `\\n[Wikipage] Failed with fetch log: ${this.#filePath}`,\n    );\n  }\n}\n\nexport function createWikipageFetchLog(logDirPath?: string): WikipageFetchLog {\n  if (logDirPath === undefined) {\n    return new SilentWikipageFetchLog();\n  }\n\n  const filePath = resolveArtifactPath({\n    category: \"wikipage\",\n    fileName: \"wikipage-fetch.jsonl\",\n    logDirPath,\n  });\n\n  return filePath === undefined\n    ? new SilentWikipageFetchLog()\n    : new FileWikipageFetchLog(filePath);\n}\n\nfunction formatEntry(entry: WikipageFetchLogEntry): Record<string, unknown> {\n  return {\n    action: entry.url.searchParams.get(\"action\") ?? null,\n    attempt: entry.attempt,\n    batch: summarizeBatch(entry.url),\n    durationMs: entry.durationMs,\n    endpoint: `${entry.url.origin}${entry.url.pathname}`,\n    host: entry.url.host,\n    ok: entry.response?.ok ?? false,\n    startedAt: new Date(entry.startedAt).toISOString(),\n    ...(entry.response === undefined\n      ? {}\n      : {\n          retryAfter: entry.response.headers.get(\"retry-after\"),\n          status: entry.response.status,\n        }),\n    ...(entry.error === undefined\n      ? {}\n      : { error: formatLogError(entry.error) }),\n    ...(entry.responseText === undefined\n      ? {}\n      : { responseText: truncateResponseText(entry.responseText) }),\n  };\n}\n\nfunction summarizeBatch(url: URL): Record<string, unknown> | undefined {\n  const ids = url.searchParams.get(\"ids\");\n  if (ids !== null) {\n    return summarizeList(\"qid\", ids.split(\"|\"));\n  }\n\n  const titles = url.searchParams.get(\"titles\");\n  if (titles !== null) {\n    return summarizeList(\"title\", titles.split(\"|\"));\n  }\n\n  const page = url.searchParams.get(\"page\");\n  if (page !== null) {\n    return summarizeList(\"page\", [page]);\n  }\n\n  return undefined;\n}\n\nfunction summarizeList(\n  kind: \"page\" | \"qid\" | \"title\",\n  values: readonly string[],\n): Record<string, unknown> {\n  const joined = values.join(\"|\");\n\n  return {\n    count: values.length,\n    hash: createHash(\"sha256\").update(joined).digest(\"hex\").slice(0, 16),\n    kind,\n    sample: values.slice(0, 5),\n  };\n}\n\nfunction formatLogError(error: unknown): Record<string, unknown> {\n  return {\n    message: formatError(error),\n    name: error instanceof Error ? error.name : typeof error,\n    ...formatErrorCause(error),\n  };\n}\n\nfunction formatErrorCause(error: unknown): Record<string, unknown> {\n  if (!(error instanceof Error) || error.cause === undefined) {\n    return {};\n  }\n\n  const cause = error.cause;\n\n  return {\n    cause:\n      cause instanceof Error\n        ? {\n            message: formatError(cause),\n            name: cause.name,\n            ...formatNodeErrorCode(cause),\n          }\n        : stringifyUnknown(cause),\n  };\n}\n\nfunction stringifyUnknown(value: unknown): string {\n  if (typeof value === \"string\") {\n    return value;\n  }\n  if (\n    typeof value === \"number\" ||\n    typeof value === \"boolean\" ||\n    typeof value === \"bigint\" ||\n    typeof value === \"symbol\"\n  ) {\n    return value.toString();\n  }\n\n  try {\n    return JSON.stringify(value);\n  } catch {\n    return \"unserializable error cause\";\n  }\n}\n\nfunction formatNodeErrorCode(error: Error): Record<string, unknown> {\n  const code = (error as { readonly code?: unknown }).code;\n\n  return typeof code === \"string\" ? { code } : {};\n}\n\nfunction truncateResponseText(text: string): string {\n  if (text.length <= MAX_RESPONSE_TEXT_LENGTH) {\n    return text;\n  }\n\n  return `${text.slice(0, MAX_RESPONSE_TEXT_LENGTH)}\\n[truncated ${text.length - MAX_RESPONSE_TEXT_LENGTH} chars]`;\n}\n"],"mappings":"AAAA,SAAS,kBAAkB;AAC3B,SAAS,kBAAkB;AAE3B,SAAS,WAAW,2BAA2B;AAC/C,SAAS,mBAAmB;AAE5B,MAAM,2BAA2B;AAkBjC,MAAM,uBAAmD;AAAA,EACvC,WAAW;AAAA,EAEpB,SAAwB;AAC7B,WAAO,QAAQ,QAAQ;AAAA,EACzB;AAAA,EAEO,aAAmB;AACxB,WAAO;AAAA,EACT;AACF;AAEA,MAAM,qBAAiD;AAAA,EAC5C;AAAA,EACT,gBAAgB;AAAA,EAET,YAAY,UAAkB;AACnC,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,IAAW,WAAmB;AAC5B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAa,OAAO,OAA6C;AAC/D,UAAM;AAAA,MACJ,KAAK;AAAA,MACL,GAAG,KAAK,UAAU,YAAY,KAAK,CAAC,CAAC;AAAA;AAAA,MACrC;AAAA,IACF;AAAA,EACF;AAAA,EAEO,aAAmB;AACxB,QAAI,KAAK,eAAe;AACtB;AAAA,IACF;AACA,SAAK,gBAAgB;AAErB,cAAU,EAAE,WAAW,WAAW,CAAC,EAAE;AAAA,MACnC;AAAA,oCAAuC,KAAK,SAAS;AAAA,IACvD;AAAA,EACF;AACF;AAEO,SAAS,uBAAuB,YAAuC;AAC5E,MAAI,eAAe,QAAW;AAC5B,WAAO,IAAI,uBAAuB;AAAA,EACpC;AAEA,QAAM,WAAW,oBAAoB;AAAA,IACnC,UAAU;AAAA,IACV,UAAU;AAAA,IACV;AAAA,EACF,CAAC;AAED,SAAO,aAAa,SAChB,IAAI,uBAAuB,IAC3B,IAAI,qBAAqB,QAAQ;AACvC;AAEA,SAAS,YAAY,OAAuD;AAC1E,SAAO;AAAA,IACL,QAAQ,MAAM,IAAI,aAAa,IAAI,QAAQ,KAAK;AAAA,IAChD,SAAS,MAAM;AAAA,IACf,OAAO,eAAe,MAAM,GAAG;AAAA,IAC/B,YAAY,MAAM;AAAA,IAClB,UAAU,GAAG,MAAM,IAAI,MAAM,GAAG,MAAM,IAAI,QAAQ;AAAA,IAClD,MAAM,MAAM,IAAI;AAAA,IAChB,IAAI,MAAM,UAAU,MAAM;AAAA,IAC1B,WAAW,IAAI,KAAK,MAAM,SAAS,EAAE,YAAY;AAAA,IACjD,GAAI,MAAM,aAAa,SACnB,CAAC,IACD;AAAA,MACE,YAAY,MAAM,SAAS,QAAQ,IAAI,aAAa;AAAA,MACpD,QAAQ,MAAM,SAAS;AAAA,IACzB;AAAA,IACJ,GAAI,MAAM,UAAU,SAChB,CAAC,IACD,EAAE,OAAO,eAAe,MAAM,KAAK,EAAE;AAAA,IACzC,GAAI,MAAM,iBAAiB,SACvB,CAAC,IACD,EAAE,cAAc,qBAAqB,MAAM,YAAY,EAAE;AAAA,EAC/D;AACF;AAEA,SAAS,eAAe,KAA+C;AACrE,QAAM,MAAM,IAAI,aAAa,IAAI,KAAK;AACtC,MAAI,QAAQ,MAAM;AAChB,WAAO,cAAc,OAAO,IAAI,MAAM,GAAG,CAAC;AAAA,EAC5C;AAEA,QAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;AAC5C,MAAI,WAAW,MAAM;AACnB,WAAO,cAAc,SAAS,OAAO,MAAM,GAAG,CAAC;AAAA,EACjD;AAEA,QAAM,OAAO,IAAI,aAAa,IAAI,MAAM;AACxC,MAAI,SAAS,MAAM;AACjB,WAAO,cAAc,QAAQ,CAAC,IAAI,CAAC;AAAA,EACrC;AAEA,SAAO;AACT;AAEA,SAAS,cACP,MACA,QACyB;AACzB,QAAM,SAAS,OAAO,KAAK,GAAG;AAE9B,SAAO;AAAA,IACL,OAAO,OAAO;AAAA,IACd,MAAM,WAAW,QAAQ,EAAE,OAAO,MAAM,EAAE,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE;AAAA,IACnE;AAAA,IACA,QAAQ,OAAO,MAAM,GAAG,CAAC;AAAA,EAC3B;AACF;AAEA,SAAS,eAAe,OAAyC;AAC/D,SAAO;AAAA,IACL,SAAS,YAAY,KAAK;AAAA,IAC1B,MAAM,iBAAiB,QAAQ,MAAM,OAAO,OAAO;AAAA,IACnD,GAAG,iBAAiB,KAAK;AAAA,EAC3B;AACF;AAEA,SAAS,iBAAiB,OAAyC;AACjE,MAAI,EAAE,iBAAiB,UAAU,MAAM,UAAU,QAAW;AAC1D,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,QAAQ,MAAM;AAEpB,SAAO;AAAA,IACL,OACE,iBAAiB,QACb;AAAA,MACE,SAAS,YAAY,KAAK;AAAA,MAC1B,MAAM,MAAM;AAAA,MACZ,GAAG,oBAAoB,KAAK;AAAA,IAC9B,IACA,iBAAiB,KAAK;AAAA,EAC9B;AACF;AAEA,SAAS,iBAAiB,OAAwB;AAChD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AACA,MACE,OAAO,UAAU,YACjB,OAAO,UAAU,aACjB,OAAO,UAAU,YACjB,OAAO,UAAU,UACjB;AACA,WAAO,MAAM,SAAS;AAAA,EACxB;AAEA,MAAI;AACF,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,oBAAoB,OAAuC;AAClE,QAAM,OAAQ,MAAsC;AAEpD,SAAO,OAAO,SAAS,WAAW,EAAE,KAAK,IAAI,CAAC;AAChD;AAEA,SAAS,qBAAqB,MAAsB;AAClD,MAAI,KAAK,UAAU,0BAA0B;AAC3C,WAAO;AAAA,EACT;AAEA,SAAO,GAAG,KAAK,MAAM,GAAG,wBAAwB,CAAC;AAAA,aAAgB,KAAK,SAAS,wBAAwB;AACzG;","names":[]}

package/dist/wikipage/index.d.ts

@@ -0,0 +1,14 @@
+export { WikipageCache, runWikipageCacheGc } from './cache.js';
+export { CreateDisambiguationProfileNormalizerOptions, createDisambiguationProfileNormalizer } from './normalizer.js';
+export { RateLimiter, parseRetryAfterMs } from './rate-limiter.js';
+export { WikipageResolver } from './resolver.js';
+export { WikimediaClient } from './wikimedia-client.js';
+export { CachedDisambiguationRecord, CachedPageRecord, CachedQidRecord, DisambiguationExpansion, DisambiguationLinkedQid, DisambiguationMeaningPriority, DisambiguationPageText, DisambiguationProfile, DisambiguationProfileMeaning, DisambiguationProfileNormalizer, DisambiguationProfileNormalizerInput, QidResolution, WikipageResolveProgress, WikipageResolveProgressDetail, WikipageResolveProgressReporter, WikipageResolverOptions, WikipageSitelink } from './types.js';
+import '../document/database.js';
+import 'sqlite3';
+import '../gc/types.js';
+import '../guaranteed/types.js';
+import '../llm/types.js';
+import 'ai';
+import 'zod';
+import './fetch-log.js';

package/dist/wikipage/index.js

@@ -0,0 +1,17 @@
+import { runWikipageCacheGc, WikipageCache } from "./cache.js";
+import {
+  createDisambiguationProfileNormalizer
+} from "./normalizer.js";
+import { RateLimiter, parseRetryAfterMs } from "./rate-limiter.js";
+import { WikipageResolver } from "./resolver.js";
+import { WikimediaClient } from "./wikimedia-client.js";
+export {
+  RateLimiter,
+  WikimediaClient,
+  WikipageCache,
+  WikipageResolver,
+  createDisambiguationProfileNormalizer,
+  parseRetryAfterMs,
+  runWikipageCacheGc
+};
+//# sourceMappingURL=index.js.map

package/dist/wikipage/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/wikipage/index.ts"],"sourcesContent":["export { runWikipageCacheGc, WikipageCache } from \"./cache.js\";\nexport {\n  createDisambiguationProfileNormalizer,\n  type CreateDisambiguationProfileNormalizerOptions,\n} from \"./normalizer.js\";\nexport { RateLimiter, parseRetryAfterMs } from \"./rate-limiter.js\";\nexport { WikipageResolver } from \"./resolver.js\";\nexport { WikimediaClient } from \"./wikimedia-client.js\";\nexport type {\n  CachedDisambiguationRecord,\n  CachedPageRecord,\n  CachedQidRecord,\n  DisambiguationExpansion,\n  DisambiguationLinkedQid,\n  DisambiguationPageText,\n  DisambiguationMeaningPriority,\n  DisambiguationProfile,\n  DisambiguationProfileMeaning,\n  DisambiguationProfileNormalizer,\n  DisambiguationProfileNormalizerInput,\n  QidResolution,\n  WikipageResolveProgress,\n  WikipageResolveProgressDetail,\n  WikipageResolveProgressReporter,\n  WikipageResolverOptions,\n  WikipageSitelink,\n} from \"./types.js\";\n"],"mappings":"AAAA,SAAS,oBAAoB,qBAAqB;AAClD;AAAA,EACE;AAAA,OAEK;AACP,SAAS,aAAa,yBAAyB;AAC/C,SAAS,wBAAwB;AACjC,SAAS,uBAAuB;","names":[]}