wiki-graph-core 0.3.1

package/dist/wikipage/normalizer.d.ts

@@ -0,0 +1,14 @@
+import { GuaranteedRequest } from '../guaranteed/types.js';
+import { DisambiguationProfileNormalizer } from './types.js';
+import '../llm/types.js';
+import 'ai';
+import 'zod';
+
+interface CreateDisambiguationProfileNormalizerOptions {
+    readonly maxMeanings?: number;
+    readonly maxRetries?: number;
+    readonly request: GuaranteedRequest;
+}
+declare function createDisambiguationProfileNormalizer(options: CreateDisambiguationProfileNormalizerOptions): DisambiguationProfileNormalizer;
+
+export { type CreateDisambiguationProfileNormalizerOptions, createDisambiguationProfileNormalizer };

package/dist/wikipage/normalizer.js

@@ -0,0 +1,147 @@
+import { z } from "zod";
+import {
+  ParsedJsonError,
+  RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE,
+  requestGuaranteedJson
+} from "../guaranteed/index.js";
+const profileMeaningSchema = z.object({
+  category: z.string().min(1).optional(),
+  information: z.string(),
+  name: z.string().min(1),
+  priority: z.enum(["primary", "secondary", "other"]),
+  qid: z.string().regex(/^Q[1-9]\d*$/u)
+}).strict();
+const profileSchema = z.object({
+  meanings: z.array(profileMeaningSchema),
+  sourceQid: z.string().regex(/^Q[1-9]\d*$/u),
+  surface: z.string().min(1).optional()
+}).strict();
+function createDisambiguationProfileNormalizer(options) {
+  return async (input) => await requestGuaranteedJson({
+    messages: buildNormalizerMessages(input, options.maxMeanings ?? 24),
+    parse: (profile) => parseDisambiguationProfile(input, normalizeProfile(profile)),
+    request: options.request,
+    responseIntentClassifierPrompt: RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE,
+    schema: profileSchema,
+    ...options.maxRetries === void 0 ? {} : { maxRetries: options.maxRetries }
+  });
+}
+function normalizeProfile(profile) {
+  return {
+    meanings: profile.meanings.map((meaning) => ({
+      ...meaning.category === void 0 ? {} : { category: meaning.category },
+      information: meaning.information,
+      name: meaning.name,
+      priority: meaning.priority,
+      qid: meaning.qid
+    })),
+    sourceQid: profile.sourceQid,
+    ...profile.surface === void 0 ? {} : { surface: profile.surface }
+  };
+}
+function parseDisambiguationProfile(input, profile) {
+  const pageQidLinks = new Set(input.pageQidLinks.map((item) => item.qid));
+  const issues = [];
+  const seen = /* @__PURE__ */ new Set();
+  const meanings = [];
+  if (profile.sourceQid !== input.sourceQid) {
+    issues.push(
+      `sourceQid must be ${input.sourceQid}, but got ${profile.sourceQid}.`
+    );
+  }
+  for (const meaning of profile.meanings) {
+    if (!pageQidLinks.has(meaning.qid)) {
+      issues.push(
+        `Meaning "${meaning.name}" selected qid ${meaning.qid}, but it is not present in the input disambiguation page links.`
+      );
+      continue;
+    }
+    if (seen.has(meaning.qid)) {
+      issues.push(
+        `Duplicate meaning qid ${meaning.qid}. Return each QID once.`
+      );
+      continue;
+    }
+    seen.add(meaning.qid);
+    meanings.push(meaning);
+  }
+  if (meanings.length === 0) {
+    issues.push("Return at least one meaning from the pageQidLinks list.");
+  }
+  if (issues.length > 0) {
+    throw new ParsedJsonError(issues);
+  }
+  return {
+    meanings,
+    sourceQid: input.sourceQid,
+    ...profile.surface === void 0 ? {} : { surface: profile.surface }
+  };
+}
+function buildNormalizerMessages(input, maxMeanings) {
+  return [
+    {
+      role: "system",
+      content: [
+        "You normalize Wikipedia disambiguation pages for entity grounding.",
+        "Return JSON only.",
+        "Do not invent QIDs, pages, people, places, works, or meanings.",
+        "Only use QIDs from the pageQidLinks list.",
+        "The information field must only copy or summarize text present on the disambiguation page itself.",
+        "Do not use Wikidata descriptions or external knowledge to fill information.",
+        "Prefer common and context-independent meanings over exhaustive long-tail lists."
+      ].join("\n")
+    },
+    {
+      role: "user",
+      content: [
+        `Source QID: ${input.sourceQid}`,
+        input.surface === void 0 ? "" : `Surface: ${input.surface}`,
+        `Maximum meanings: ${maxMeanings}`,
+        "",
+        "Page QID links:",
+        JSON.stringify(input.pageQidLinks.map(formatPageQidLink), null, 2),
+        "",
+        "Disambiguation page text:",
+        input.pages.map(
+          (page) => `--- ${page.wiki}:${page.title} ---
+${truncatePageText(page.text)}`
+        ).join("\n\n"),
+        "",
+        "Return this JSON shape:",
+        JSON.stringify(
+          {
+            meanings: [
+              {
+                category: "person | place | organization | work | concept | event | other",
+                information: "short information copied or summarized from the disambiguation page; empty string if no information is present",
+                name: "display name",
+                priority: "primary | secondary | other",
+                qid: "QID from pageQidLinks"
+              }
+            ],
+            sourceQid: input.sourceQid,
+            ...input.surface === void 0 ? {} : { surface: input.surface }
+          },
+          null,
+          2
+        )
+      ].filter((line) => line !== "").join("\n")
+    }
+  ];
+}
+function formatPageQidLink(item) {
+  return {
+    qid: item.qid,
+    title: item.title
+  };
+}
+function truncatePageText(text) {
+  const maxLength = 16e3;
+  return text.length <= maxLength ? text : `${text.slice(0, maxLength)}
+
+[truncated]`;
+}
+export {
+  createDisambiguationProfileNormalizer
+};
+//# sourceMappingURL=normalizer.js.map

package/dist/wikipage/normalizer.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/wikipage/normalizer.ts"],"sourcesContent":["import { z } from \"zod\";\n\nimport {\n  ParsedJsonError,\n  RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE,\n  requestGuaranteedJson,\n  type GuaranteedRequest,\n} from \"../guaranteed/index.js\";\nimport type { LLMessage } from \"../llm/index.js\";\n\nimport type {\n  DisambiguationLinkedQid,\n  DisambiguationProfile,\n  DisambiguationProfileMeaning,\n  DisambiguationProfileNormalizer,\n  DisambiguationProfileNormalizerInput,\n} from \"./types.js\";\n\nconst profileMeaningSchema = z\n  .object({\n    category: z.string().min(1).optional(),\n    information: z.string(),\n    name: z.string().min(1),\n    priority: z.enum([\"primary\", \"secondary\", \"other\"]),\n    qid: z.string().regex(/^Q[1-9]\\d*$/u),\n  })\n  .strict();\n\nconst profileSchema = z\n  .object({\n    meanings: z.array(profileMeaningSchema),\n    sourceQid: z.string().regex(/^Q[1-9]\\d*$/u),\n    surface: z.string().min(1).optional(),\n  })\n  .strict();\n\nexport interface CreateDisambiguationProfileNormalizerOptions {\n  readonly maxMeanings?: number;\n  readonly maxRetries?: number;\n  readonly request: GuaranteedRequest;\n}\n\nexport function createDisambiguationProfileNormalizer(\n  options: CreateDisambiguationProfileNormalizerOptions,\n): DisambiguationProfileNormalizer {\n  return async (input) =>\n    await requestGuaranteedJson({\n      messages: buildNormalizerMessages(input, options.maxMeanings ?? 24),\n      parse: (profile) =>\n        parseDisambiguationProfile(input, normalizeProfile(profile)),\n      request: options.request,\n      responseIntentClassifierPrompt:\n        RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE,\n      schema: profileSchema,\n      ...(options.maxRetries === undefined\n        ? {}\n        : { maxRetries: options.maxRetries }),\n    });\n}\n\nfunction normalizeProfile(\n  profile: z.infer<typeof profileSchema>,\n): DisambiguationProfile {\n  return {\n    meanings: profile.meanings.map((meaning) => ({\n      ...(meaning.category === undefined ? {} : { category: meaning.category }),\n      information: meaning.information,\n      name: meaning.name,\n      priority: meaning.priority,\n      qid: meaning.qid,\n    })),\n    sourceQid: profile.sourceQid,\n    ...(profile.surface === undefined ? {} : { surface: profile.surface }),\n  };\n}\n\nfunction parseDisambiguationProfile(\n  input: DisambiguationProfileNormalizerInput,\n  profile: DisambiguationProfile,\n): DisambiguationProfile {\n  const pageQidLinks = new Set(input.pageQidLinks.map((item) => item.qid));\n  const issues: string[] = [];\n  const seen = new Set<string>();\n  const meanings: DisambiguationProfileMeaning[] = [];\n\n  if (profile.sourceQid !== input.sourceQid) {\n    issues.push(\n      `sourceQid must be ${input.sourceQid}, but got ${profile.sourceQid}.`,\n    );\n  }\n\n  for (const meaning of profile.meanings) {\n    if (!pageQidLinks.has(meaning.qid)) {\n      issues.push(\n        `Meaning \"${meaning.name}\" selected qid ${meaning.qid}, but it is not present in the input disambiguation page links.`,\n      );\n      continue;\n    }\n    if (seen.has(meaning.qid)) {\n      issues.push(\n        `Duplicate meaning qid ${meaning.qid}. Return each QID once.`,\n      );\n      continue;\n    }\n\n    seen.add(meaning.qid);\n    meanings.push(meaning);\n  }\n\n  if (meanings.length === 0) {\n    issues.push(\"Return at least one meaning from the pageQidLinks list.\");\n  }\n  if (issues.length > 0) {\n    throw new ParsedJsonError(issues);\n  }\n\n  return {\n    meanings,\n    sourceQid: input.sourceQid,\n    ...(profile.surface === undefined ? {} : { surface: profile.surface }),\n  };\n}\n\nfunction buildNormalizerMessages(\n  input: DisambiguationProfileNormalizerInput,\n  maxMeanings: number,\n): LLMessage[] {\n  return [\n    {\n      role: \"system\",\n      content: [\n        \"You normalize Wikipedia disambiguation pages for entity grounding.\",\n        \"Return JSON only.\",\n        \"Do not invent QIDs, pages, people, places, works, or meanings.\",\n        \"Only use QIDs from the pageQidLinks list.\",\n        \"The information field must only copy or summarize text present on the disambiguation page itself.\",\n        \"Do not use Wikidata descriptions or external knowledge to fill information.\",\n        \"Prefer common and context-independent meanings over exhaustive long-tail lists.\",\n      ].join(\"\\n\"),\n    },\n    {\n      role: \"user\",\n      content: [\n        `Source QID: ${input.sourceQid}`,\n        input.surface === undefined ? \"\" : `Surface: ${input.surface}`,\n        `Maximum meanings: ${maxMeanings}`,\n        \"\",\n        \"Page QID links:\",\n        JSON.stringify(input.pageQidLinks.map(formatPageQidLink), null, 2),\n        \"\",\n        \"Disambiguation page text:\",\n        input.pages\n          .map(\n            (page) =>\n              `--- ${page.wiki}:${page.title} ---\\n${truncatePageText(page.text)}`,\n          )\n          .join(\"\\n\\n\"),\n        \"\",\n        \"Return this JSON shape:\",\n        JSON.stringify(\n          {\n            meanings: [\n              {\n                category:\n                  \"person | place | organization | work | concept | event | other\",\n                information:\n                  \"short information copied or summarized from the disambiguation page; empty string if no information is present\",\n                name: \"display name\",\n                priority: \"primary | secondary | other\",\n                qid: \"QID from pageQidLinks\",\n              },\n            ],\n            sourceQid: input.sourceQid,\n            ...(input.surface === undefined ? {} : { surface: input.surface }),\n          },\n          null,\n          2,\n        ),\n      ]\n        .filter((line) => line !== \"\")\n        .join(\"\\n\"),\n    },\n  ];\n}\n\nfunction formatPageQidLink(item: DisambiguationLinkedQid): object {\n  return {\n    qid: item.qid,\n    title: item.title,\n  };\n}\n\nfunction truncatePageText(text: string): string {\n  const maxLength = 16000;\n\n  return text.length <= maxLength\n    ? text\n    : `${text.slice(0, maxLength)}\\n\\n[truncated]`;\n}\n"],"mappings":"AAAA,SAAS,SAAS;AAElB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAWP,MAAM,uBAAuB,EAC1B,OAAO;AAAA,EACN,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,SAAS;AAAA,EACrC,aAAa,EAAE,OAAO;AAAA,EACtB,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EACtB,UAAU,EAAE,KAAK,CAAC,WAAW,aAAa,OAAO,CAAC;AAAA,EAClD,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc;AACtC,CAAC,EACA,OAAO;AAEV,MAAM,gBAAgB,EACnB,OAAO;AAAA,EACN,UAAU,EAAE,MAAM,oBAAoB;AAAA,EACtC,WAAW,EAAE,OAAO,EAAE,MAAM,cAAc;AAAA,EAC1C,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,SAAS;AACtC,CAAC,EACA,OAAO;AAQH,SAAS,sCACd,SACiC;AACjC,SAAO,OAAO,UACZ,MAAM,sBAAsB;AAAA,IAC1B,UAAU,wBAAwB,OAAO,QAAQ,eAAe,EAAE;AAAA,IAClE,OAAO,CAAC,YACN,2BAA2B,OAAO,iBAAiB,OAAO,CAAC;AAAA,IAC7D,SAAS,QAAQ;AAAA,IACjB,gCACE;AAAA,IACF,QAAQ;AAAA,IACR,GAAI,QAAQ,eAAe,SACvB,CAAC,IACD,EAAE,YAAY,QAAQ,WAAW;AAAA,EACvC,CAAC;AACL;AAEA,SAAS,iBACP,SACuB;AACvB,SAAO;AAAA,IACL,UAAU,QAAQ,SAAS,IAAI,CAAC,aAAa;AAAA,MAC3C,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,MACvE,aAAa,QAAQ;AAAA,MACrB,MAAM,QAAQ;AAAA,MACd,UAAU,QAAQ;AAAA,MAClB,KAAK,QAAQ;AAAA,IACf,EAAE;AAAA,IACF,WAAW,QAAQ;AAAA,IACnB,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,EACtE;AACF;AAEA,SAAS,2BACP,OACA,SACuB;AACvB,QAAM,eAAe,IAAI,IAAI,MAAM,aAAa,IAAI,CAAC,SAAS,KAAK,GAAG,CAAC;AACvE,QAAM,SAAmB,CAAC;AAC1B,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,WAA2C,CAAC;AAElD,MAAI,QAAQ,cAAc,MAAM,WAAW;AACzC,WAAO;AAAA,MACL,qBAAqB,MAAM,SAAS,aAAa,QAAQ,SAAS;AAAA,IACpE;AAAA,EACF;AAEA,aAAW,WAAW,QAAQ,UAAU;AACtC,QAAI,CAAC,aAAa,IAAI,QAAQ,GAAG,GAAG;AAClC,aAAO;AAAA,QACL,YAAY,QAAQ,IAAI,kBAAkB,QAAQ,GAAG;AAAA,MACvD;AACA;AAAA,IACF;AACA,QAAI,KAAK,IAAI,QAAQ,GAAG,GAAG;AACzB,aAAO;AAAA,QACL,yBAAyB,QAAQ,GAAG;AAAA,MACtC;AACA;AAAA,IACF;AAEA,SAAK,IAAI,QAAQ,GAAG;AACpB,aAAS,KAAK,OAAO;AAAA,EACvB;AAEA,MAAI,SAAS,WAAW,GAAG;AACzB,WAAO,KAAK,yDAAyD;AAAA,EACvE;AACA,MAAI,OAAO,SAAS,GAAG;AACrB,UAAM,IAAI,gBAAgB,MAAM;AAAA,EAClC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,WAAW,MAAM;AAAA,IACjB,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,EACtE;AACF;AAEA,SAAS,wBACP,OACA,aACa;AACb,SAAO;AAAA,IACL;AAAA,MACE,MAAM;AAAA,MACN,SAAS;AAAA,QACP;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAAA,IACb;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,SAAS;AAAA,QACP,eAAe,MAAM,SAAS;AAAA,QAC9B,MAAM,YAAY,SAAY,KAAK,YAAY,MAAM,OAAO;AAAA,QAC5D,qBAAqB,WAAW;AAAA,QAChC;AAAA,QACA;AAAA,QACA,KAAK,UAAU,MAAM,aAAa,IAAI,iBAAiB,GAAG,MAAM,CAAC;AAAA,QACjE;AAAA,QACA;AAAA,QACA,MAAM,MACH;AAAA,UACC,CAAC,SACC,OAAO,KAAK,IAAI,IAAI,KAAK,KAAK;AAAA,EAAS,iBAAiB,KAAK,IAAI,CAAC;AAAA,QACtE,EACC,KAAK,MAAM;AAAA,QACd;AAAA,QACA;AAAA,QACA,KAAK;AAAA,UACH;AAAA,YACE,UAAU;AAAA,cACR;AAAA,gBACE,UACE;AAAA,gBACF,aACE;AAAA,gBACF,MAAM;AAAA,gBACN,UAAU;AAAA,gBACV,KAAK;AAAA,cACP;AAAA,YACF;AAAA,YACA,WAAW,MAAM;AAAA,YACjB,GAAI,MAAM,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,MAAM,QAAQ;AAAA,UAClE;AAAA,UACA;AAAA,UACA;AAAA,QACF;AAAA,MACF,EACG,OAAO,CAAC,SAAS,SAAS,EAAE,EAC5B,KAAK,IAAI;AAAA,IACd;AAAA,EACF;AACF;AAEA,SAAS,kBAAkB,MAAuC;AAChE,SAAO;AAAA,IACL,KAAK,KAAK;AAAA,IACV,OAAO,KAAK;AAAA,EACd;AACF;AAEA,SAAS,iBAAiB,MAAsB;AAC9C,QAAM,YAAY;AAElB,SAAO,KAAK,UAAU,YAClB,OACA,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC;AAAA;AAAA;AACjC;","names":[]}

package/dist/wikipage/rate-limiter.d.ts

@@ -0,0 +1,13 @@
+interface RateLimiterOptions {
+    readonly concurrency: number;
+    readonly minRequestIntervalMs: number;
+}
+declare class RateLimiter {
+    #private;
+    constructor(options: RateLimiterOptions);
+    use<T>(operation: () => Promise<T>): Promise<T>;
+    blockFor(ms: number): void;
+}
+declare function parseRetryAfterMs(value: string | null): number | undefined;
+
+export { RateLimiter, type RateLimiterOptions, parseRetryAfterMs };

package/dist/wikipage/rate-limiter.js

@@ -0,0 +1,84 @@
+class RateLimiter {
+  #concurrency;
+  #minRequestIntervalMs;
+  #active = 0;
+  #blockedUntil = 0;
+  #lastStartedAt = 0;
+  #queue = [];
+  constructor(options) {
+    this.#concurrency = Math.max(1, Math.floor(options.concurrency));
+    this.#minRequestIntervalMs = Math.max(
+      0,
+      Math.floor(options.minRequestIntervalMs)
+    );
+  }
+  async use(operation) {
+    await this.#acquire();
+    try {
+      return await operation();
+    } finally {
+      this.#active -= 1;
+      this.#releaseNext();
+    }
+  }
+  blockFor(ms) {
+    if (!Number.isFinite(ms) || ms <= 0) {
+      return;
+    }
+    this.#blockedUntil = Math.max(this.#blockedUntil, Date.now() + ms);
+  }
+  async #acquire() {
+    await new Promise((resolve) => {
+      this.#queue.push(resolve);
+      this.#releaseNext();
+    });
+    await this.#waitForTurn();
+    this.#active += 1;
+    this.#lastStartedAt = Date.now();
+  }
+  async #waitForTurn() {
+    const blockedDelayMs = Math.max(0, this.#blockedUntil - Date.now());
+    const intervalDelayMs = Math.max(
+      0,
+      this.#lastStartedAt + this.#minRequestIntervalMs - Date.now()
+    );
+    const delayMs = Math.max(blockedDelayMs, intervalDelayMs);
+    if (delayMs > 0) {
+      await delay(delayMs);
+    }
+  }
+  #releaseNext() {
+    if (this.#active >= this.#concurrency) {
+      return;
+    }
+    const resolve = this.#queue.shift();
+    if (resolve === void 0) {
+      return;
+    }
+    resolve();
+  }
+}
+async function delay(ms) {
+  await new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+function parseRetryAfterMs(value) {
+  if (value === null || value.trim() === "") {
+    return void 0;
+  }
+  const seconds = Number(value);
+  if (Number.isFinite(seconds)) {
+    return Math.max(0, seconds * 1e3);
+  }
+  const retryAt = Date.parse(value);
+  if (Number.isNaN(retryAt)) {
+    return void 0;
+  }
+  return Math.max(0, retryAt - Date.now());
+}
+export {
+  RateLimiter,
+  parseRetryAfterMs
+};
+//# sourceMappingURL=rate-limiter.js.map

package/dist/wikipage/rate-limiter.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/wikipage/rate-limiter.ts"],"sourcesContent":["export interface RateLimiterOptions {\n  readonly concurrency: number;\n  readonly minRequestIntervalMs: number;\n}\n\nexport class RateLimiter {\n  readonly #concurrency: number;\n  readonly #minRequestIntervalMs: number;\n  #active = 0;\n  #blockedUntil = 0;\n  #lastStartedAt = 0;\n  readonly #queue: Array<() => void> = [];\n\n  public constructor(options: RateLimiterOptions) {\n    this.#concurrency = Math.max(1, Math.floor(options.concurrency));\n    this.#minRequestIntervalMs = Math.max(\n      0,\n      Math.floor(options.minRequestIntervalMs),\n    );\n  }\n\n  public async use<T>(operation: () => Promise<T>): Promise<T> {\n    await this.#acquire();\n\n    try {\n      return await operation();\n    } finally {\n      this.#active -= 1;\n      this.#releaseNext();\n    }\n  }\n\n  public blockFor(ms: number): void {\n    if (!Number.isFinite(ms) || ms <= 0) {\n      return;\n    }\n\n    this.#blockedUntil = Math.max(this.#blockedUntil, Date.now() + ms);\n  }\n\n  async #acquire(): Promise<void> {\n    await new Promise<void>((resolve) => {\n      this.#queue.push(resolve);\n      this.#releaseNext();\n    });\n\n    await this.#waitForTurn();\n    this.#active += 1;\n    this.#lastStartedAt = Date.now();\n  }\n\n  async #waitForTurn(): Promise<void> {\n    const blockedDelayMs = Math.max(0, this.#blockedUntil - Date.now());\n    const intervalDelayMs = Math.max(\n      0,\n      this.#lastStartedAt + this.#minRequestIntervalMs - Date.now(),\n    );\n    const delayMs = Math.max(blockedDelayMs, intervalDelayMs);\n\n    if (delayMs > 0) {\n      await delay(delayMs);\n    }\n  }\n\n  #releaseNext(): void {\n    if (this.#active >= this.#concurrency) {\n      return;\n    }\n\n    const resolve = this.#queue.shift();\n\n    if (resolve === undefined) {\n      return;\n    }\n\n    resolve();\n  }\n}\n\nasync function delay(ms: number): Promise<void> {\n  await new Promise<void>((resolve) => {\n    setTimeout(resolve, ms);\n  });\n}\n\nexport function parseRetryAfterMs(value: string | null): number | undefined {\n  if (value === null || value.trim() === \"\") {\n    return undefined;\n  }\n\n  const seconds = Number(value);\n\n  if (Number.isFinite(seconds)) {\n    return Math.max(0, seconds * 1000);\n  }\n\n  const retryAt = Date.parse(value);\n\n  if (Number.isNaN(retryAt)) {\n    return undefined;\n  }\n\n  return Math.max(0, retryAt - Date.now());\n}\n"],"mappings":"AAKO,MAAM,YAAY;AAAA,EACd;AAAA,EACA;AAAA,EACT,UAAU;AAAA,EACV,gBAAgB;AAAA,EAChB,iBAAiB;AAAA,EACR,SAA4B,CAAC;AAAA,EAE/B,YAAY,SAA6B;AAC9C,SAAK,eAAe,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,WAAW,CAAC;AAC/D,SAAK,wBAAwB,KAAK;AAAA,MAChC;AAAA,MACA,KAAK,MAAM,QAAQ,oBAAoB;AAAA,IACzC;AAAA,EACF;AAAA,EAEA,MAAa,IAAO,WAAyC;AAC3D,UAAM,KAAK,SAAS;AAEpB,QAAI;AACF,aAAO,MAAM,UAAU;AAAA,IACzB,UAAE;AACA,WAAK,WAAW;AAChB,WAAK,aAAa;AAAA,IACpB;AAAA,EACF;AAAA,EAEO,SAAS,IAAkB;AAChC,QAAI,CAAC,OAAO,SAAS,EAAE,KAAK,MAAM,GAAG;AACnC;AAAA,IACF;AAEA,SAAK,gBAAgB,KAAK,IAAI,KAAK,eAAe,KAAK,IAAI,IAAI,EAAE;AAAA,EACnE;AAAA,EAEA,MAAM,WAA0B;AAC9B,UAAM,IAAI,QAAc,CAAC,YAAY;AACnC,WAAK,OAAO,KAAK,OAAO;AACxB,WAAK,aAAa;AAAA,IACpB,CAAC;AAED,UAAM,KAAK,aAAa;AACxB,SAAK,WAAW;AAChB,SAAK,iBAAiB,KAAK,IAAI;AAAA,EACjC;AAAA,EAEA,MAAM,eAA8B;AAClC,UAAM,iBAAiB,KAAK,IAAI,GAAG,KAAK,gBAAgB,KAAK,IAAI,CAAC;AAClE,UAAM,kBAAkB,KAAK;AAAA,MAC3B;AAAA,MACA,KAAK,iBAAiB,KAAK,wBAAwB,KAAK,IAAI;AAAA,IAC9D;AACA,UAAM,UAAU,KAAK,IAAI,gBAAgB,eAAe;AAExD,QAAI,UAAU,GAAG;AACf,YAAM,MAAM,OAAO;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,eAAqB;AACnB,QAAI,KAAK,WAAW,KAAK,cAAc;AACrC;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,OAAO,MAAM;AAElC,QAAI,YAAY,QAAW;AACzB;AAAA,IACF;AAEA,YAAQ;AAAA,EACV;AACF;AAEA,eAAe,MAAM,IAA2B;AAC9C,QAAM,IAAI,QAAc,CAAC,YAAY;AACnC,eAAW,SAAS,EAAE;AAAA,EACxB,CAAC;AACH;AAEO,SAAS,kBAAkB,OAA0C;AAC1E,MAAI,UAAU,QAAQ,MAAM,KAAK,MAAM,IAAI;AACzC,WAAO;AAAA,EACT;AAEA,QAAM,UAAU,OAAO,KAAK;AAE5B,MAAI,OAAO,SAAS,OAAO,GAAG;AAC5B,WAAO,KAAK,IAAI,GAAG,UAAU,GAAI;AAAA,EACnC;AAEA,QAAM,UAAU,KAAK,MAAM,KAAK;AAEhC,MAAI,OAAO,MAAM,OAAO,GAAG;AACzB,WAAO;AAAA,EACT;AAEA,SAAO,KAAK,IAAI,GAAG,UAAU,KAAK,IAAI,CAAC;AACzC;","names":[]}

package/dist/wikipage/resolver.d.ts

@@ -0,0 +1,26 @@
+import { WikipageCache } from './cache.js';
+import { WikimediaClient } from './wikimedia-client.js';
+import { DisambiguationProfileNormalizer, WikipageResolveProgressReporter, WikipageResolverOptions, QidResolution } from './types.js';
+import '../document/database.js';
+import 'sqlite3';
+import '../gc/types.js';
+import './fetch-log.js';
+
+declare class WikipageResolver {
+    #private;
+    constructor(input: {
+        readonly cache: WikipageCache;
+        readonly client: WikimediaClient;
+        readonly language: string;
+        readonly maxBatchSize: number;
+        readonly normalizer: DisambiguationProfileNormalizer | undefined;
+        readonly ownsCache: boolean;
+        readonly progress: WikipageResolveProgressReporter | undefined;
+        readonly wiki: string;
+    });
+    static open(options?: WikipageResolverOptions): Promise<WikipageResolver>;
+    close(): Promise<void>;
+    resolveQids(qids: readonly string[]): Promise<readonly QidResolution[]>;
+}
+
+export { WikipageResolver };

package/dist/wikipage/resolver.js

@@ -0,0 +1,325 @@
+import { WikipageCache } from "./cache.js";
+import { createWikipageFetchLog } from "./fetch-log.js";
+import {
+  WikimediaClient,
+  replaceTitleUriWithQidUri
+} from "./wikimedia-client.js";
+const DEFAULT_CONCURRENCY = 3;
+const DEFAULT_LANGUAGE = "zh";
+const DEFAULT_MAX_BATCH_SIZE = 50;
+const DEFAULT_MIN_REQUEST_INTERVAL_MS = 100;
+const DEFAULT_RETRY_BASE_DELAY_MS = 1e3;
+const DEFAULT_RETRY_TIMES = 3;
+const DEFAULT_USER_AGENT = "WikiGraph/0.3 (https://github.com/oomol-lab/wiki-graph)";
+class WikipageResolver {
+  #cache;
+  #client;
+  #maxBatchSize;
+  #language;
+  #normalizer;
+  #ownsCache;
+  #progress;
+  #wiki;
+  constructor(input) {
+    this.#cache = input.cache;
+    this.#client = input.client;
+    this.#language = input.language;
+    this.#maxBatchSize = input.maxBatchSize;
+    this.#normalizer = input.normalizer;
+    this.#ownsCache = input.ownsCache;
+    this.#progress = input.progress;
+    this.#wiki = input.wiki;
+  }
+  static async open(options = {}) {
+    const language = normalizeLanguage(options.language);
+    const wiki = options.wiki ?? `${language}wiki`;
+    const cache = await WikipageCache.open(options.cacheDatabasePath);
+    return new WikipageResolver({
+      cache,
+      client: new WikimediaClient({
+        concurrency: options.concurrency ?? DEFAULT_CONCURRENCY,
+        language,
+        requestLog: createWikipageFetchLog(options.logDirPath),
+        minRequestIntervalMs: options.minRequestIntervalMs ?? DEFAULT_MIN_REQUEST_INTERVAL_MS,
+        retryBaseDelayMs: options.retryBaseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS,
+        retryTimes: options.retryTimes ?? DEFAULT_RETRY_TIMES,
+        userAgent: options.userAgent ?? DEFAULT_USER_AGENT,
+        wiki,
+        ...options.fetch === void 0 ? {} : { fetch: options.fetch }
+      }),
+      language,
+      maxBatchSize: normalizeBatchSize(options.maxBatchSize),
+      normalizer: options.normalizer,
+      ownsCache: true,
+      progress: options.progress,
+      wiki
+    });
+  }
+  async close() {
+    if (this.#ownsCache) {
+      await this.#cache.close();
+    }
+  }
+  async resolveQids(qids) {
+    const normalizedQids = normalizeQids(qids);
+    const qidRecords = await this.#resolveQidRecords(normalizedQids);
+    const disambiguationRecords = await this.#resolveDisambiguations(
+      [...qidRecords.values()].filter(hasDisambiguationPage)
+    );
+    return normalizedQids.map((qid) => {
+      const record = qidRecords.get(qid);
+      if (record === void 0) {
+        return {
+          isDisambiguation: false,
+          qid
+        };
+      }
+      const disambiguation = disambiguationRecords.get(qid);
+      const disambiguationPages = disambiguation?.pages ?? [];
+      const primarySitelink = pickPrimarySitelink(record.sitelinks, this.#wiki);
+      return {
+        ...record.description === void 0 ? {} : { description: record.description },
+        ...disambiguation === void 0 ? {} : { disambiguation: toDisambiguationExpansion(disambiguation) },
+        ...disambiguationPages.length === 0 ? {} : { disambiguationPages },
+        isDisambiguation: disambiguationPages.length > 0,
+        ...record.label === void 0 ? {} : { label: record.label },
+        qid,
+        ...primarySitelink === void 0 ? {} : { sitelink: primarySitelink },
+        ...record.sitelinks.length === 0 ? {} : { sitelinks: record.sitelinks.map(toSitelink) }
+      };
+    });
+  }
+  async #resolveQidRecords(qids) {
+    const cached = new Map(await this.#cache.getQids(qids, this.#language));
+    const missing = qids.filter((qid) => !cached.has(qid));
+    let resolvedQids = cached.size;
+    let resolvedEntities = 0;
+    await this.#reportProgress("qid", resolvedQids, qids.length);
+    for (const batch of chunk(missing, this.#maxBatchSize)) {
+      const entityInfos = await this.#client.getEntities(batch);
+      resolvedEntities += batch.length;
+      await this.#reportProgress("entity", resolvedEntities, missing.length);
+      const pageInfos = await this.#fetchPageInfos(entityInfos);
+      const now = (/* @__PURE__ */ new Date()).toISOString();
+      const records = batch.map(
+        (qid) => createQidRecord(qid, entityInfos.get(qid), pageInfos, now)
+      );
+      await this.#cache.putQids(records, this.#language);
+      for (const record of records) {
+        cached.set(record.qid, record);
+      }
+      resolvedQids += batch.length;
+      await this.#reportProgress("qid", resolvedQids, qids.length);
+    }
+    return cached;
+  }
+  async #fetchPageInfos(entityInfos) {
+    const results = /* @__PURE__ */ new Map();
+    const titlesByWiki = /* @__PURE__ */ new Map();
+    for (const entity of entityInfos.values()) {
+      for (const sitelink of entity.sitelinks) {
+        const titles = titlesByWiki.get(sitelink.wiki) ?? /* @__PURE__ */ new Set();
+        titles.add(sitelink.title);
+        titlesByWiki.set(sitelink.wiki, titles);
+      }
+    }
+    for (const [wiki, titles] of titlesByWiki) {
+      for (const batch of chunk([...titles], this.#maxBatchSize)) {
+        for (const [title, page] of await this.#client.getPagesByTitles(
+          batch,
+          wiki
+        )) {
+          results.set(pageKey(wiki, title), page);
+        }
+        await this.#reportProgress(
+          "page",
+          results.size,
+          countTitleSetItems(titlesByWiki)
+        );
+      }
+    }
+    return results;
+  }
+  async #resolveDisambiguations(records) {
+    const qids = records.map((record) => record.qid);
+    const cached = new Map(
+      await this.#cache.getDisambiguations(qids, this.#wiki)
+    );
+    const missing = records.filter((record) => !cached.has(record.qid));
+    let resolvedPages = cached.size;
+    await this.#reportProgress(
+      "disambiguation-page",
+      resolvedPages,
+      qids.length
+    );
+    for (const record of missing) {
+      const expansion = await this.#expandDisambiguation(record);
+      await this.#cache.putDisambiguations([expansion], this.#wiki);
+      cached.set(record.qid, expansion);
+      resolvedPages += 1;
+      await this.#reportProgress(
+        "disambiguation-page",
+        resolvedPages,
+        qids.length
+      );
+    }
+    return cached;
+  }
+  async #expandDisambiguation(record) {
+    const pages = [];
+    for (const page of record.sitelinks.filter(
+      (item) => item.isDisambiguation
+    )) {
+      const parsedPage = await this.#client.parseDisambiguationPage(
+        page.title,
+        page.wiki
+      );
+      const linkedPageInfos = /* @__PURE__ */ new Map();
+      for (const batch of chunk(parsedPage.linkedTitles, this.#maxBatchSize)) {
+        for (const [title, linkedPage] of await this.#client.getPagesByTitles(
+          batch,
+          page.wiki
+        )) {
+          linkedPageInfos.set(title, linkedPage);
+        }
+        await this.#reportProgress(
+          "linked-page",
+          linkedPageInfos.size,
+          parsedPage.linkedTitles.length
+        );
+      }
+      const titleToQid = /* @__PURE__ */ new Map();
+      const linkedQids2 = [];
+      for (const title of parsedPage.linkedTitles) {
+        const qid = linkedPageInfos.get(title)?.wikibaseItem;
+        if (qid === void 0) {
+          continue;
+        }
+        titleToQid.set(title, qid);
+        linkedQids2.push({ qid, title });
+      }
+      pages.push({
+        linkedQids: linkedQids2,
+        ...parsedPage.pageId === void 0 ? {} : { pageId: parsedPage.pageId },
+        text: replaceTitleUriWithQidUri(parsedPage.text, titleToQid),
+        title: parsedPage.title,
+        wiki: parsedPage.wiki
+      });
+    }
+    const linkedQids = mergeLinkedQids(pages);
+    const profile = this.#normalizer === void 0 || linkedQids.length === 0 ? void 0 : await this.#normalizer({
+      pageQidLinks: linkedQids,
+      pages,
+      sourceQid: record.qid,
+      ...record.label === void 0 ? {} : { surface: record.label }
+    });
+    return {
+      checkedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      disambiguationQid: record.qid,
+      pages,
+      ...profile === void 0 ? {} : { profile }
+    };
+  }
+  async #reportProgress(detail, done, total) {
+    await this.#progress?.({
+      detail,
+      done: Math.min(Math.max(0, done), Math.max(0, total)),
+      total: Math.max(0, total)
+    });
+  }
+}
+function createQidRecord(qid, entityInfo, pageInfos, now) {
+  return {
+    checkedAt: now,
+    ...entityInfo?.description === void 0 ? {} : { description: entityInfo.description },
+    ...entityInfo?.label === void 0 ? {} : { label: entityInfo.label },
+    qid,
+    sitelinks: entityInfo?.sitelinks.flatMap((sitelink) => {
+      const page = pageInfos.get(pageKey(sitelink.wiki, sitelink.title));
+      if (page === void 0) {
+        return [];
+      }
+      return [
+        {
+          isDisambiguation: page.isDisambiguation,
+          ...page.pageId === void 0 ? {} : { pageId: page.pageId },
+          title: page.title,
+          wiki: page.wiki
+        }
+      ];
+    }) ?? [],
+    updatedAt: now
+  };
+}
+function toDisambiguationExpansion(record) {
+  return {
+    checkedAt: record.checkedAt,
+    disambiguationQid: record.disambiguationQid,
+    linkedQids: mergeLinkedQids(record.pages),
+    pages: record.pages,
+    ...record.profile === void 0 ? {} : { profile: record.profile }
+  };
+}
+function mergeLinkedQids(pages) {
+  const results = /* @__PURE__ */ new Map();
+  for (const page of pages) {
+    for (const item of page.linkedQids) {
+      results.set(item.qid, item);
+    }
+  }
+  return [...results.values()];
+}
+function countTitleSetItems(input) {
+  let count = 0;
+  for (const titles of input.values()) {
+    count += titles.size;
+  }
+  return count;
+}
+function hasDisambiguationPage(record) {
+  return record.sitelinks.some((page) => page.isDisambiguation);
+}
+function pickPrimarySitelink(pages, wiki) {
+  const preferred = pages.find((page) => page.wiki === wiki) ?? pages[0];
+  return preferred === void 0 ? void 0 : toSitelink(preferred);
+}
+function toSitelink(page) {
+  return {
+    title: page.title,
+    wiki: page.wiki
+  };
+}
+function pageKey(wiki, title) {
+  return `${wiki}:${title}`;
+}
+function normalizeQids(qids) {
+  return [
+    ...new Set(
+      qids.map((qid) => qid.trim().toUpperCase()).filter((qid) => /^Q[1-9]\d*$/u.test(qid))
+    )
+  ];
+}
+function normalizeLanguage(value) {
+  const normalized = value?.trim().toLowerCase();
+  if (normalized === "en" || normalized === "zh") {
+    return normalized;
+  }
+  return DEFAULT_LANGUAGE;
+}
+function normalizeBatchSize(value) {
+  if (value === void 0 || !Number.isFinite(value)) {
+    return DEFAULT_MAX_BATCH_SIZE;
+  }
+  return Math.max(1, Math.floor(value));
+}
+function chunk(items, size) {
+  const batches = [];
+  for (let index = 0; index < items.length; index += size) {
+    batches.push(items.slice(index, index + size));
+  }
+  return batches;
+}
+export {
+  WikipageResolver
+};
+//# sourceMappingURL=resolver.js.map