wiki-graph-core 0.3.1 → 0.4.0

package/dist/external/wikipage/resolver.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/external/wikipage/resolver.ts"],"sourcesContent":["import { WikipageCache } from \"./cache.js\";\nimport { createWikipageFetchLog } from \"./fetch-log.js\";\nimport { GuaranteedRequestFailureError } from \"../guaranteed/index.js\";\nimport {\n  WikimediaClient,\n  replaceTitleUriWithQidUri,\n  type SupportedWiki,\n  type WikiPageInfo,\n  type WikidataEntityInfo,\n} from \"./wikimedia-client/index.js\";\nimport type {\n  CachedDisambiguationRecord,\n  CachedPageRecord,\n  CachedQidRecord,\n  DisambiguationExpansion,\n  DisambiguationLinkedQid,\n  DisambiguationPageText,\n  DisambiguationProfileNormalizer,\n  EnrichmentStore,\n  QidResolution,\n  WikiClient,\n  WikipageResolveProgressReporter,\n  WikipageResolverOptions,\n} from \"./types.js\";\n\nconst DEFAULT_CONCURRENCY = 3;\nconst DEFAULT_LANGUAGE = \"zh\";\nconst DEFAULT_MAX_BATCH_SIZE = 50;\nconst DEFAULT_MIN_REQUEST_INTERVAL_MS = 100;\nconst DEFAULT_RETRY_BASE_DELAY_MS = 1_000;\nconst DEFAULT_RETRY_TIMES = 3;\nconst DEFAULT_USER_AGENT =\n  \"WikiGraph/0.3 (https://github.com/oomol-lab/wiki-graph)\";\nconst DISAMBIGUATION_PROFILE_ERROR_RETRY_DELAY_MS = 6 * 60 * 60 * 1_000;\n\nexport class WikipageResolver {\n  readonly #client: WikiClient;\n  readonly #maxBatchSize: number;\n  readonly #language: string;\n  readonly #normalizer: DisambiguationProfileNormalizer | undefined;\n  readonly #ownsStore: boolean;\n  readonly #progress: WikipageResolveProgressReporter | undefined;\n  readonly #store: EnrichmentStore;\n  readonly #wiki: string;\n\n  public constructor(input: {\n    readonly client: WikiClient;\n    readonly language: string;\n    readonly maxBatchSize: number;\n    readonly normalizer: DisambiguationProfileNormalizer | undefined;\n    readonly ownsStore?: boolean;\n    readonly progress: WikipageResolveProgressReporter | undefined;\n    readonly store: EnrichmentStore;\n    readonly wiki: string;\n  }) {\n    this.#client = input.client;\n    this.#language = input.language;\n    this.#maxBatchSize = input.maxBatchSize;\n    this.#normalizer = input.normalizer;\n    this.#ownsStore = input.ownsStore ?? false;\n    this.#progress = input.progress;\n    this.#store = input.store;\n    this.#wiki = input.wiki;\n  }\n\n  public static async open(\n    options: WikipageResolverOptions = {},\n  ): Promise<WikipageResolver> {\n    const language = normalizeLanguage(options.language);\n    const wiki = options.wiki ?? `${language}wiki`;\n    const cache = await WikipageCache.open(options.cacheDatabasePath);\n\n    return new WikipageResolver({\n      client: new WikimediaClient({\n        concurrency: options.concurrency ?? DEFAULT_CONCURRENCY,\n        language,\n        requestLog: createWikipageFetchLog(options.logDirPath),\n        minRequestIntervalMs:\n          options.minRequestIntervalMs ?? DEFAULT_MIN_REQUEST_INTERVAL_MS,\n        retryBaseDelayMs:\n          options.retryBaseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS,\n        retryTimes: options.retryTimes ?? DEFAULT_RETRY_TIMES,\n        userAgent: options.userAgent ?? DEFAULT_USER_AGENT,\n        wiki,\n        ...(options.fetch === undefined ? {} : { fetch: options.fetch }),\n      }),\n      language,\n      maxBatchSize: normalizeBatchSize(options.maxBatchSize),\n      normalizer: options.normalizer,\n      ownsStore: true,\n      progress: options.progress,\n      store: cache,\n      wiki,\n    });\n  }\n\n  public async close(): Promise<void> {\n    if (this.#ownsStore && isClosableStore(this.#store)) {\n      await this.#store.close();\n    }\n  }\n\n  public async resolveQids(\n    qids: readonly string[],\n  ): Promise<readonly QidResolution[]> {\n    const normalizedQids = normalizeQids(qids);\n    const qidRecords = await this.#resolveQidRecords(normalizedQids);\n    const disambiguationRecords = await this.#resolveDisambiguations(\n      [...qidRecords.values()].filter(hasDisambiguationPage),\n    );\n\n    return normalizedQids.map((qid) => {\n      const record = qidRecords.get(qid);\n\n      if (record === undefined) {\n        return {\n          isDisambiguation: false,\n          qid,\n        };\n      }\n\n      const disambiguation = disambiguationRecords.get(qid);\n      const disambiguationPages = disambiguation?.pages ?? [];\n      const primarySitelink = pickPrimarySitelink(record.sitelinks, this.#wiki);\n\n      return {\n        ...(record.description === undefined\n          ? {}\n          : { description: record.description }),\n        ...(disambiguation === undefined\n          ? {}\n          : { disambiguation: toDisambiguationExpansion(disambiguation) }),\n        ...(disambiguationPages.length === 0 ? {} : { disambiguationPages }),\n        isDisambiguation: disambiguationPages.length > 0,\n        ...(record.label === undefined ? {} : { label: record.label }),\n        qid,\n        ...(primarySitelink === undefined ? {} : { sitelink: primarySitelink }),\n        ...(record.sitelinks.length === 0\n          ? {}\n          : { sitelinks: record.sitelinks.map(toSitelink) }),\n      };\n    });\n  }\n\n  async #resolveQidRecords(\n    qids: readonly string[],\n  ): Promise<ReadonlyMap<string, CachedQidRecord>> {\n    const cached = new Map(await this.#store.getQids(qids, this.#language));\n    const missing = qids.filter((qid) => !cached.has(qid));\n    let resolvedQids = cached.size;\n    let resolvedEntities = 0;\n\n    await this.#reportProgress(\"qid\", resolvedQids, qids.length);\n\n    for (const batch of chunk(missing, this.#maxBatchSize)) {\n      const entityInfos = await this.#client.getEntities(batch);\n      resolvedEntities += batch.length;\n      await this.#reportProgress(\"entity\", resolvedEntities, missing.length);\n      const pageInfos = await this.#fetchPageInfos(entityInfos);\n      const now = new Date().toISOString();\n      const records = batch.map((qid) =>\n        createQidRecord(qid, entityInfos.get(qid), pageInfos, now),\n      );\n\n      await this.#store.putQids(records, this.#language);\n      for (const record of records) {\n        cached.set(record.qid, record);\n      }\n      resolvedQids += batch.length;\n      await this.#reportProgress(\"qid\", resolvedQids, qids.length);\n    }\n\n    return cached;\n  }\n\n  async #fetchPageInfos(\n    entityInfos: ReadonlyMap<string, WikidataEntityInfo>,\n  ): Promise<ReadonlyMap<string, WikiPageInfo>> {\n    const results = new Map<string, WikiPageInfo>();\n    const titlesByWiki = new Map<SupportedWiki, Set<string>>();\n\n    for (const entity of entityInfos.values()) {\n      for (const sitelink of entity.sitelinks) {\n        const titles = titlesByWiki.get(sitelink.wiki) ?? new Set<string>();\n\n        titles.add(sitelink.title);\n        titlesByWiki.set(sitelink.wiki, titles);\n      }\n    }\n\n    for (const [wiki, titles] of titlesByWiki) {\n      for (const batch of chunk([...titles], this.#maxBatchSize)) {\n        for (const [title, page] of await this.#client.getPagesByTitles(\n          batch,\n          wiki,\n        )) {\n          results.set(pageKey(wiki, title), page);\n        }\n        await this.#reportProgress(\n          \"page\",\n          results.size,\n          countTitleSetItems(titlesByWiki),\n        );\n      }\n    }\n\n    return results;\n  }\n\n  async #resolveDisambiguations(\n    records: readonly CachedQidRecord[],\n  ): Promise<ReadonlyMap<string, CachedDisambiguationRecord>> {\n    const qids = records.map((record) => record.qid);\n    const cached = new Map(\n      await this.#store.getDisambiguations(qids, this.#wiki),\n    );\n    const missing = records.filter((record) => !cached.has(record.qid));\n    let resolvedPages = cached.size;\n\n    await this.#reportProgress(\n      \"disambiguation-page\",\n      resolvedPages,\n      qids.length,\n    );\n\n    for (const record of missing) {\n      const expansion = await this.#expandDisambiguation(record);\n\n      await this.#store.putDisambiguations([expansion], this.#wiki);\n      cached.set(record.qid, expansion);\n      resolvedPages += 1;\n      await this.#reportProgress(\n        \"disambiguation-page\",\n        resolvedPages,\n        qids.length,\n      );\n    }\n\n    return cached;\n  }\n\n  async #expandDisambiguation(\n    record: CachedQidRecord,\n  ): Promise<CachedDisambiguationRecord> {\n    const pages: DisambiguationPageText[] = [];\n\n    for (const page of record.sitelinks.filter(\n      (item) => item.isDisambiguation,\n    )) {\n      const parsedPage = await this.#client.parseDisambiguationPage(\n        page.title,\n        page.wiki,\n      );\n      const linkedPageInfos = new Map<string, WikiPageInfo>();\n\n      for (const batch of chunk(parsedPage.linkedTitles, this.#maxBatchSize)) {\n        for (const [title, linkedPage] of await this.#client.getPagesByTitles(\n          batch,\n          page.wiki,\n        )) {\n          linkedPageInfos.set(title, linkedPage);\n        }\n        await this.#reportProgress(\n          \"linked-page\",\n          linkedPageInfos.size,\n          parsedPage.linkedTitles.length,\n        );\n      }\n\n      const titleToQid = new Map<string, string>();\n      const linkedQids: DisambiguationLinkedQid[] = [];\n\n      for (const title of parsedPage.linkedTitles) {\n        const qid = linkedPageInfos.get(title)?.wikibaseItem;\n\n        if (qid === undefined) {\n          continue;\n        }\n\n        titleToQid.set(title, qid);\n        linkedQids.push({ qid, title });\n      }\n\n      pages.push({\n        linkedQids,\n        ...(parsedPage.pageId === undefined\n          ? {}\n          : { pageId: parsedPage.pageId }),\n        text: replaceTitleUriWithQidUri(parsedPage.text, titleToQid),\n        title: parsedPage.title,\n        wiki: parsedPage.wiki,\n      });\n    }\n\n    const linkedQids = mergeLinkedQids(pages);\n    const profileResult = await this.#normalizeDisambiguationProfile(\n      record,\n      pages,\n      linkedQids,\n    );\n\n    return {\n      checkedAt: new Date().toISOString(),\n      disambiguationQid: record.qid,\n      pages,\n      ...profileResult,\n    };\n  }\n\n  async #normalizeDisambiguationProfile(\n    record: CachedQidRecord,\n    pages: readonly DisambiguationPageText[],\n    linkedQids: readonly DisambiguationLinkedQid[],\n  ): Promise<Pick<CachedDisambiguationRecord, \"profile\" | \"profileError\">> {\n    if (this.#normalizer === undefined || linkedQids.length === 0) {\n      return {};\n    }\n\n    try {\n      return {\n        profile: await this.#normalizer({\n          pageQidLinks: linkedQids,\n          pages,\n          sourceQid: record.qid,\n          ...(record.label === undefined ? {} : { surface: record.label }),\n        }),\n      };\n    } catch (error) {\n      if (!(error instanceof GuaranteedRequestFailureError)) {\n        throw error;\n      }\n\n      const failedAt = new Date();\n      const retryAfter = new Date(\n        failedAt.getTime() + DISAMBIGUATION_PROFILE_ERROR_RETRY_DELAY_MS,\n      );\n\n      return {\n        profileError: {\n          failedAt: failedAt.toISOString(),\n          message: error.message,\n          retryAfter: retryAfter.toISOString(),\n        },\n      };\n    }\n  }\n\n  async #reportProgress(\n    detail: Parameters<WikipageResolveProgressReporter>[0][\"detail\"],\n    done: number,\n    total: number,\n  ): Promise<void> {\n    await this.#progress?.({\n      detail,\n      done: Math.min(Math.max(0, done), Math.max(0, total)),\n      total: Math.max(0, total),\n    });\n  }\n}\n\nfunction createQidRecord(\n  qid: string,\n  entityInfo: WikidataEntityInfo | undefined,\n  pageInfos: ReadonlyMap<string, WikiPageInfo>,\n  now: string,\n): CachedQidRecord {\n  return {\n    checkedAt: now,\n    ...(entityInfo?.description === undefined\n      ? {}\n      : { description: entityInfo.description }),\n    ...(entityInfo?.label === undefined ? {} : { label: entityInfo.label }),\n    qid,\n    sitelinks:\n      entityInfo?.sitelinks.flatMap((sitelink): CachedPageRecord[] => {\n        const page = pageInfos.get(pageKey(sitelink.wiki, sitelink.title));\n\n        if (page === undefined) {\n          return [];\n        }\n\n        return [\n          {\n            isDisambiguation: page.isDisambiguation,\n            ...(page.pageId === undefined ? {} : { pageId: page.pageId }),\n            title: page.title,\n            wiki: page.wiki,\n          },\n        ];\n      }) ?? [],\n    updatedAt: now,\n  };\n}\n\nfunction toDisambiguationExpansion(\n  record: CachedDisambiguationRecord,\n): DisambiguationExpansion {\n  return {\n    checkedAt: record.checkedAt,\n    disambiguationQid: record.disambiguationQid,\n    linkedQids: mergeLinkedQids(record.pages),\n    pages: record.pages,\n    ...(record.profile === undefined ? {} : { profile: record.profile }),\n  };\n}\n\nfunction mergeLinkedQids(\n  pages: readonly DisambiguationPageText[],\n): readonly DisambiguationLinkedQid[] {\n  const results = new Map<string, DisambiguationLinkedQid>();\n\n  for (const page of pages) {\n    for (const item of page.linkedQids) {\n      results.set(item.qid, item);\n    }\n  }\n\n  return [...results.values()];\n}\n\nfunction countTitleSetItems(\n  input: ReadonlyMap<SupportedWiki, Set<string>>,\n): number {\n  let count = 0;\n\n  for (const titles of input.values()) {\n    count += titles.size;\n  }\n\n  return count;\n}\n\nfunction hasDisambiguationPage(record: CachedQidRecord): boolean {\n  return record.sitelinks.some((page) => page.isDisambiguation);\n}\n\nfunction pickPrimarySitelink(\n  pages: readonly CachedPageRecord[],\n  wiki: string,\n): { readonly title: string; readonly wiki: string } | undefined {\n  const preferred = pages.find((page) => page.wiki === wiki) ?? pages[0];\n\n  return preferred === undefined ? undefined : toSitelink(preferred);\n}\n\nfunction toSitelink(page: CachedPageRecord): {\n  readonly title: string;\n  readonly wiki: string;\n} {\n  return {\n    title: page.title,\n    wiki: page.wiki,\n  };\n}\n\nfunction isClosableStore(\n  store: EnrichmentStore,\n): store is EnrichmentStore & { readonly close: () => Promise<void> } {\n  return \"close\" in store && typeof store.close === \"function\";\n}\n\nfunction pageKey(wiki: SupportedWiki, title: string): string {\n  return `${wiki}:${title}`;\n}\n\nfunction normalizeQids(qids: readonly string[]): readonly string[] {\n  return [\n    ...new Set(\n      qids\n        .map((qid) => qid.trim().toUpperCase())\n        .filter((qid) => /^Q[1-9]\\d*$/u.test(qid)),\n    ),\n  ];\n}\n\nfunction normalizeLanguage(value: string | undefined): string {\n  const normalized = value?.trim().toLowerCase();\n\n  if (normalized === \"en\" || normalized === \"zh\") {\n    return normalized;\n  }\n\n  return DEFAULT_LANGUAGE;\n}\n\nfunction normalizeBatchSize(value: number | undefined): number {\n  if (value === undefined || !Number.isFinite(value)) {\n    return DEFAULT_MAX_BATCH_SIZE;\n  }\n\n  return Math.max(1, Math.floor(value));\n}\n\nfunction chunk<T>(items: readonly T[], size: number): readonly T[][] {\n  const batches: T[][] = [];\n\n  for (let index = 0; index < items.length; index += size) {\n    batches.push(items.slice(index, index + size));\n  }\n\n  return batches;\n}\n"],"mappings":"AAAA,SAAS,qBAAqB;AAC9B,SAAS,8BAA8B;AACvC,SAAS,qCAAqC;AAC9C;AAAA,EACE;AAAA,EACA;AAAA,OAIK;AAgBP,MAAM,sBAAsB;AAC5B,MAAM,mBAAmB;AACzB,MAAM,yBAAyB;AAC/B,MAAM,kCAAkC;AACxC,MAAM,8BAA8B;AACpC,MAAM,sBAAsB;AAC5B,MAAM,qBACJ;AACF,MAAM,8CAA8C,IAAI,KAAK,KAAK;AAE3D,MAAM,iBAAiB;AAAA,EACnB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEF,YAAY,OAShB;AACD,SAAK,UAAU,MAAM;AACrB,SAAK,YAAY,MAAM;AACvB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,cAAc,MAAM;AACzB,SAAK,aAAa,MAAM,aAAa;AACrC,SAAK,YAAY,MAAM;AACvB,SAAK,SAAS,MAAM;AACpB,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,aAAoB,KAClB,UAAmC,CAAC,GACT;AAC3B,UAAM,WAAW,kBAAkB,QAAQ,QAAQ;AACnD,UAAM,OAAO,QAAQ,QAAQ,GAAG,QAAQ;AACxC,UAAM,QAAQ,MAAM,cAAc,KAAK,QAAQ,iBAAiB;AAEhE,WAAO,IAAI,iBAAiB;AAAA,MAC1B,QAAQ,IAAI,gBAAgB;AAAA,QAC1B,aAAa,QAAQ,eAAe;AAAA,QACpC;AAAA,QACA,YAAY,uBAAuB,QAAQ,UAAU;AAAA,QACrD,sBACE,QAAQ,wBAAwB;AAAA,QAClC,kBACE,QAAQ,oBAAoB;AAAA,QAC9B,YAAY,QAAQ,cAAc;AAAA,QAClC,WAAW,QAAQ,aAAa;AAAA,QAChC;AAAA,QACA,GAAI,QAAQ,UAAU,SAAY,CAAC,IAAI,EAAE,OAAO,QAAQ,MAAM;AAAA,MAChE,CAAC;AAAA,MACD;AAAA,MACA,cAAc,mBAAmB,QAAQ,YAAY;AAAA,MACrD,YAAY,QAAQ;AAAA,MACpB,WAAW;AAAA,MACX,UAAU,QAAQ;AAAA,MAClB,OAAO;AAAA,MACP;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAa,QAAuB;AAClC,QAAI,KAAK,cAAc,gBAAgB,KAAK,MAAM,GAAG;AACnD,YAAM,KAAK,OAAO,MAAM;AAAA,IAC1B;AAAA,EACF;AAAA,EAEA,MAAa,YACX,MACmC;AACnC,UAAM,iBAAiB,cAAc,IAAI;AACzC,UAAM,aAAa,MAAM,KAAK,mBAAmB,cAAc;AAC/D,UAAM,wBAAwB,MAAM,KAAK;AAAA,MACvC,CAAC,GAAG,WAAW,OAAO,CAAC,EAAE,OAAO,qBAAqB;AAAA,IACvD;AAEA,WAAO,eAAe,IAAI,CAAC,QAAQ;AACjC,YAAM,SAAS,WAAW,IAAI,GAAG;AAEjC,UAAI,WAAW,QAAW;AACxB,eAAO;AAAA,UACL,kBAAkB;AAAA,UAClB;AAAA,QACF;AAAA,MACF;AAEA,YAAM,iBAAiB,sBAAsB,IAAI,GAAG;AACpD,YAAM,sBAAsB,gBAAgB,SAAS,CAAC;AACtD,YAAM,kBAAkB,oBAAoB,OAAO,WAAW,KAAK,KAAK;AAExE,aAAO;AAAA,QACL,GAAI,OAAO,gBAAgB,SACvB,CAAC,IACD,EAAE,aAAa,OAAO,YAAY;AAAA,QACtC,GAAI,mBAAmB,SACnB,CAAC,IACD,EAAE,gBAAgB,0BAA0B,cAAc,EAAE;AAAA,QAChE,GAAI,oBAAoB,WAAW,IAAI,CAAC,IAAI,EAAE,oBAAoB;AAAA,QAClE,kBAAkB,oBAAoB,SAAS;AAAA,QAC/C,GAAI,OAAO,UAAU,SAAY,CAAC,IAAI,EAAE,OAAO,OAAO,MAAM;AAAA,QAC5D;AAAA,QACA,GAAI,oBAAoB,SAAY,CAAC,IAAI,EAAE,UAAU,gBAAgB;AAAA,QACrE,GAAI,OAAO,UAAU,WAAW,IAC5B,CAAC,IACD,EAAE,WAAW,OAAO,UAAU,IAAI,UAAU,EAAE;AAAA,MACpD;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,mBACJ,MAC+C;AAC/C,UAAM,SAAS,IAAI,IAAI,MAAM,KAAK,OAAO,QAAQ,MAAM,KAAK,SAAS,CAAC;AACtE,UAAM,UAAU,KAAK,OAAO,CAAC,QAAQ,CAAC,OAAO,IAAI,GAAG,CAAC;AACrD,QAAI,eAAe,OAAO;AAC1B,QAAI,mBAAmB;AAEvB,UAAM,KAAK,gBAAgB,OAAO,cAAc,KAAK,MAAM;AAE3D,eAAW,SAAS,MAAM,SAAS,KAAK,aAAa,GAAG;AACtD,YAAM,cAAc,MAAM,KAAK,QAAQ,YAAY,KAAK;AACxD,0BAAoB,MAAM;AAC1B,YAAM,KAAK,gBAAgB,UAAU,kBAAkB,QAAQ,MAAM;AACrE,YAAM,YAAY,MAAM,KAAK,gBAAgB,WAAW;AACxD,YAAM,OAAM,oBAAI,KAAK,GAAE,YAAY;AACnC,YAAM,UAAU,MAAM;AAAA,QAAI,CAAC,QACzB,gBAAgB,KAAK,YAAY,IAAI,GAAG,GAAG,WAAW,GAAG;AAAA,MAC3D;AAEA,YAAM,KAAK,OAAO,QAAQ,SAAS,KAAK,SAAS;AACjD,iBAAW,UAAU,SAAS;AAC5B,eAAO,IAAI,OAAO,KAAK,MAAM;AAAA,MAC/B;AACA,sBAAgB,MAAM;AACtB,YAAM,KAAK,gBAAgB,OAAO,cAAc,KAAK,MAAM;AAAA,IAC7D;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBACJ,aAC4C;AAC5C,UAAM,UAAU,oBAAI,IAA0B;AAC9C,UAAM,eAAe,oBAAI,IAAgC;AAEzD,eAAW,UAAU,YAAY,OAAO,GAAG;AACzC,iBAAW,YAAY,OAAO,WAAW;AACvC,cAAM,SAAS,aAAa,IAAI,SAAS,IAAI,KAAK,oBAAI,IAAY;AAElE,eAAO,IAAI,SAAS,KAAK;AACzB,qBAAa,IAAI,SAAS,MAAM,MAAM;AAAA,MACxC;AAAA,IACF;AAEA,eAAW,CAAC,MAAM,MAAM,KAAK,cAAc;AACzC,iBAAW,SAAS,MAAM,CAAC,GAAG,MAAM,GAAG,KAAK,aAAa,GAAG;AAC1D,mBAAW,CAAC,OAAO,IAAI,KAAK,MAAM,KAAK,QAAQ;AAAA,UAC7C;AAAA,UACA;AAAA,QACF,GAAG;AACD,kBAAQ,IAAI,QAAQ,MAAM,KAAK,GAAG,IAAI;AAAA,QACxC;AACA,cAAM,KAAK;AAAA,UACT;AAAA,UACA,QAAQ;AAAA,UACR,mBAAmB,YAAY;AAAA,QACjC;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,wBACJ,SAC0D;AAC1D,UAAM,OAAO,QAAQ,IAAI,CAAC,WAAW,OAAO,GAAG;AAC/C,UAAM,SAAS,IAAI;AAAA,MACjB,MAAM,KAAK,OAAO,mBAAmB,MAAM,KAAK,KAAK;AAAA,IACvD;AACA,UAAM,UAAU,QAAQ,OAAO,CAAC,WAAW,CAAC,OAAO,IAAI,OAAO,GAAG,CAAC;AAClE,QAAI,gBAAgB,OAAO;AAE3B,UAAM,KAAK;AAAA,MACT;AAAA,MACA;AAAA,MACA,KAAK;AAAA,IACP;AAEA,eAAW,UAAU,SAAS;AAC5B,YAAM,YAAY,MAAM,KAAK,sBAAsB,MAAM;AAEzD,YAAM,KAAK,OAAO,mBAAmB,CAAC,SAAS,GAAG,KAAK,KAAK;AAC5D,aAAO,IAAI,OAAO,KAAK,SAAS;AAChC,uBAAiB;AACjB,YAAM,KAAK;AAAA,QACT;AAAA,QACA;AAAA,QACA,KAAK;AAAA,MACP;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,sBACJ,QACqC;AACrC,UAAM,QAAkC,CAAC;AAEzC,eAAW,QAAQ,OAAO,UAAU;AAAA,MAClC,CAAC,SAAS,KAAK;AAAA,IACjB,GAAG;AACD,YAAM,aAAa,MAAM,KAAK,QAAQ;AAAA,QACpC,KAAK;AAAA,QACL,KAAK;AAAA,MACP;AACA,YAAM,kBAAkB,oBAAI,IAA0B;AAEtD,iBAAW,SAAS,MAAM,WAAW,cAAc,KAAK,aAAa,GAAG;AACtE,mBAAW,CAAC,OAAO,UAAU,KAAK,MAAM,KAAK,QAAQ;AAAA,UACnD;AAAA,UACA,KAAK;AAAA,QACP,GAAG;AACD,0BAAgB,IAAI,OAAO,UAAU;AAAA,QACvC;AACA,cAAM,KAAK;AAAA,UACT;AAAA,UACA,gBAAgB;AAAA,UAChB,WAAW,aAAa;AAAA,QAC1B;AAAA,MACF;AAEA,YAAM,aAAa,oBAAI,IAAoB;AAC3C,YAAMA,cAAwC,CAAC;AAE/C,iBAAW,SAAS,WAAW,cAAc;AAC3C,cAAM,MAAM,gBAAgB,IAAI,KAAK,GAAG;AAExC,YAAI,QAAQ,QAAW;AACrB;AAAA,QACF;AAEA,mBAAW,IAAI,OAAO,GAAG;AACzB,QAAAA,YAAW,KAAK,EAAE,KAAK,MAAM,CAAC;AAAA,MAChC;AAEA,YAAM,KAAK;AAAA,QACT,YAAAA;AAAA,QACA,GAAI,WAAW,WAAW,SACtB,CAAC,IACD,EAAE,QAAQ,WAAW,OAAO;AAAA,QAChC,MAAM,0BAA0B,WAAW,MAAM,UAAU;AAAA,QAC3D,OAAO,WAAW;AAAA,QAClB,MAAM,WAAW;AAAA,MACnB,CAAC;AAAA,IACH;AAEA,UAAM,aAAa,gBAAgB,KAAK;AACxC,UAAM,gBAAgB,MAAM,KAAK;AAAA,MAC/B;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,WAAO;AAAA,MACL,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,mBAAmB,OAAO;AAAA,MAC1B;AAAA,MACA,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,gCACJ,QACA,OACA,YACuE;AACvE,QAAI,KAAK,gBAAgB,UAAa,WAAW,WAAW,GAAG;AAC7D,aAAO,CAAC;AAAA,IACV;AAEA,QAAI;AACF,aAAO;AAAA,QACL,SAAS,MAAM,KAAK,YAAY;AAAA,UAC9B,cAAc;AAAA,UACd;AAAA,UACA,WAAW,OAAO;AAAA,UAClB,GAAI,OAAO,UAAU,SAAY,CAAC,IAAI,EAAE,SAAS,OAAO,MAAM;AAAA,QAChE,CAAC;AAAA,MACH;AAAA,IACF,SAAS,OAAO;AACd,UAAI,EAAE,iBAAiB,gCAAgC;AACrD,cAAM;AAAA,MACR;AAEA,YAAM,WAAW,oBAAI,KAAK;AAC1B,YAAM,aAAa,IAAI;AAAA,QACrB,SAAS,QAAQ,IAAI;AAAA,MACvB;AAEA,aAAO;AAAA,QACL,cAAc;AAAA,UACZ,UAAU,SAAS,YAAY;AAAA,UAC/B,SAAS,MAAM;AAAA,UACf,YAAY,WAAW,YAAY;AAAA,QACrC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,gBACJ,QACA,MACA,OACe;AACf,UAAM,KAAK,YAAY;AAAA,MACrB;AAAA,MACA,MAAM,KAAK,IAAI,KAAK,IAAI,GAAG,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AAAA,MACpD,OAAO,KAAK,IAAI,GAAG,KAAK;AAAA,IAC1B,CAAC;AAAA,EACH;AACF;AAEA,SAAS,gBACP,KACA,YACA,WACA,KACiB;AACjB,SAAO;AAAA,IACL,WAAW;AAAA,IACX,GAAI,YAAY,gBAAgB,SAC5B,CAAC,IACD,EAAE,aAAa,WAAW,YAAY;AAAA,IAC1C,GAAI,YAAY,UAAU,SAAY,CAAC,IAAI,EAAE,OAAO,WAAW,MAAM;AAAA,IACrE;AAAA,IACA,WACE,YAAY,UAAU,QAAQ,CAAC,aAAiC;AAC9D,YAAM,OAAO,UAAU,IAAI,QAAQ,SAAS,MAAM,SAAS,KAAK,CAAC;AAEjE,UAAI,SAAS,QAAW;AACtB,eAAO,CAAC;AAAA,MACV;AAEA,aAAO;AAAA,QACL;AAAA,UACE,kBAAkB,KAAK;AAAA,UACvB,GAAI,KAAK,WAAW,SAAY,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO;AAAA,UAC3D,OAAO,KAAK;AAAA,UACZ,MAAM,KAAK;AAAA,QACb;AAAA,MACF;AAAA,IACF,CAAC,KAAK,CAAC;AAAA,IACT,WAAW;AAAA,EACb;AACF;AAEA,SAAS,0BACP,QACyB;AACzB,SAAO;AAAA,IACL,WAAW,OAAO;AAAA,IAClB,mBAAmB,OAAO;AAAA,IAC1B,YAAY,gBAAgB,OAAO,KAAK;AAAA,IACxC,OAAO,OAAO;AAAA,IACd,GAAI,OAAO,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,OAAO,QAAQ;AAAA,EACpE;AACF;AAEA,SAAS,gBACP,OACoC;AACpC,QAAM,UAAU,oBAAI,IAAqC;AAEzD,aAAW,QAAQ,OAAO;AACxB,eAAW,QAAQ,KAAK,YAAY;AAClC,cAAQ,IAAI,KAAK,KAAK,IAAI;AAAA,IAC5B;AAAA,EACF;AAEA,SAAO,CAAC,GAAG,QAAQ,OAAO,CAAC;AAC7B;AAEA,SAAS,mBACP,OACQ;AACR,MAAI,QAAQ;AAEZ,aAAW,UAAU,MAAM,OAAO,GAAG;AACnC,aAAS,OAAO;AAAA,EAClB;AAEA,SAAO;AACT;AAEA,SAAS,sBAAsB,QAAkC;AAC/D,SAAO,OAAO,UAAU,KAAK,CAAC,SAAS,KAAK,gBAAgB;AAC9D;AAEA,SAAS,oBACP,OACA,MAC+D;AAC/D,QAAM,YAAY,MAAM,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,KAAK,MAAM,CAAC;AAErE,SAAO,cAAc,SAAY,SAAY,WAAW,SAAS;AACnE;AAEA,SAAS,WAAW,MAGlB;AACA,SAAO;AAAA,IACL,OAAO,KAAK;AAAA,IACZ,MAAM,KAAK;AAAA,EACb;AACF;AAEA,SAAS,gBACP,OACoE;AACpE,SAAO,WAAW,SAAS,OAAO,MAAM,UAAU;AACpD;AAEA,SAAS,QAAQ,MAAqB,OAAuB;AAC3D,SAAO,GAAG,IAAI,IAAI,KAAK;AACzB;AAEA,SAAS,cAAc,MAA4C;AACjE,SAAO;AAAA,IACL,GAAG,IAAI;AAAA,MACL,KACG,IAAI,CAAC,QAAQ,IAAI,KAAK,EAAE,YAAY,CAAC,EACrC,OAAO,CAAC,QAAQ,eAAe,KAAK,GAAG,CAAC;AAAA,IAC7C;AAAA,EACF;AACF;AAEA,SAAS,kBAAkB,OAAmC;AAC5D,QAAM,aAAa,OAAO,KAAK,EAAE,YAAY;AAE7C,MAAI,eAAe,QAAQ,eAAe,MAAM;AAC9C,WAAO;AAAA,EACT;AAEA,SAAO;AACT;AAEA,SAAS,mBAAmB,OAAmC;AAC7D,MAAI,UAAU,UAAa,CAAC,OAAO,SAAS,KAAK,GAAG;AAClD,WAAO;AAAA,EACT;AAEA,SAAO,KAAK,IAAI,GAAG,KAAK,MAAM,KAAK,CAAC;AACtC;AAEA,SAAS,MAAS,OAAqB,MAA8B;AACnE,QAAM,UAAiB,CAAC;AAExB,WAAS,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS,MAAM;AACvD,YAAQ,KAAK,MAAM,MAAM,OAAO,QAAQ,IAAI,CAAC;AAAA,EAC/C;AAEA,SAAO;AACT;","names":["linkedQids"]}

package/dist/external/wikipage/types.d.ts

@@ -0,0 +1,118 @@
+import { WikidataEntityInfo, SupportedWiki, WikiPageInfo, ParsedDisambiguationPage } from './wikimedia-client/wiki.js';
+import './fetch-log.js';
+
+interface WikipageResolverOptions {
+    readonly cacheDatabasePath?: string;
+    readonly concurrency?: number;
+    readonly fetch?: typeof fetch;
+    readonly language?: string;
+    readonly logDirPath?: string;
+    readonly maxBatchSize?: number;
+    readonly minRequestIntervalMs?: number;
+    readonly normalizer?: DisambiguationProfileNormalizer;
+    readonly progress?: WikipageResolveProgressReporter;
+    readonly retryBaseDelayMs?: number;
+    readonly retryTimes?: number;
+    readonly userAgent?: string;
+    readonly wiki?: string;
+}
+interface WikiClient {
+    /** Runtime adapter for reading Wikimedia/Wikidata data; local CLI uses WikimediaClient. */
+    readonly getEntities: (qids: readonly string[]) => Promise<ReadonlyMap<string, WikidataEntityInfo>>;
+    readonly getPagesByTitles: (titles: readonly string[], wiki?: SupportedWiki) => Promise<ReadonlyMap<string, WikiPageInfo>>;
+    readonly parseDisambiguationPage: (title: string, wiki: SupportedWiki) => Promise<ParsedDisambiguationPage>;
+}
+interface EnrichmentStore {
+    /** Runtime adapter for enrichment cache reads and writes; local CLI uses WikipageCache. */
+    readonly getQids: (qids: readonly string[], language: string) => Promise<ReadonlyMap<string, CachedQidRecord>>;
+    readonly putQids: (records: readonly CachedQidRecord[], language: string) => Promise<void>;
+    readonly getDisambiguations: (qids: readonly string[], wiki: string) => Promise<ReadonlyMap<string, CachedDisambiguationRecord>>;
+    readonly putDisambiguations: (records: readonly CachedDisambiguationRecord[], wiki: string) => Promise<void>;
+}
+type WikipageResolveProgressDetail = "disambiguation-page" | "entity" | "linked-page" | "page" | "qid";
+interface WikipageResolveProgress {
+    readonly detail: WikipageResolveProgressDetail;
+    readonly done: number;
+    readonly total: number;
+}
+type WikipageResolveProgressReporter = (event: WikipageResolveProgress) => Promise<void> | void;
+interface QidResolution {
+    readonly description?: string;
+    readonly disambiguation?: DisambiguationExpansion;
+    readonly disambiguationPages?: readonly DisambiguationPageText[];
+    readonly isDisambiguation: boolean;
+    readonly label?: string;
+    readonly qid: string;
+    readonly sitelink?: WikipageSitelink;
+    readonly sitelinks?: readonly WikipageSitelink[];
+}
+interface WikipageSitelink {
+    readonly title: string;
+    readonly wiki: string;
+}
+interface DisambiguationExpansion {
+    readonly checkedAt: string;
+    readonly disambiguationQid: string;
+    readonly linkedQids: readonly DisambiguationLinkedQid[];
+    readonly pages: readonly DisambiguationPageText[];
+    readonly profile?: DisambiguationProfile;
+}
+interface DisambiguationPageText {
+    readonly linkedQids: readonly DisambiguationLinkedQid[];
+    readonly pageId?: number;
+    readonly text: string;
+    readonly title: string;
+    readonly wiki: "enwiki" | "zhwiki";
+}
+interface DisambiguationLinkedQid {
+    readonly qid: string;
+    readonly title: string;
+}
+interface CachedQidRecord {
+    readonly checkedAt: string;
+    readonly description?: string;
+    readonly label?: string;
+    readonly qid: string;
+    readonly sitelinks: readonly CachedPageRecord[];
+    readonly updatedAt: string;
+}
+interface CachedDisambiguationRecord {
+    readonly checkedAt: string;
+    readonly disambiguationQid: string;
+    readonly pages: readonly DisambiguationPageText[];
+    readonly profileError?: DisambiguationProfileError;
+    readonly profile?: DisambiguationProfile;
+}
+interface DisambiguationProfileError {
+    readonly failedAt: string;
+    readonly message: string;
+    readonly retryAfter: string;
+}
+interface CachedPageRecord {
+    readonly isDisambiguation: boolean;
+    readonly pageId?: number;
+    readonly title: string;
+    readonly wiki: "enwiki" | "zhwiki";
+}
+type DisambiguationMeaningPriority = "other" | "primary" | "secondary";
+interface DisambiguationProfile {
+    readonly meanings: readonly DisambiguationProfileMeaning[];
+    readonly sourceQid: string;
+    readonly surface?: string;
+}
+interface DisambiguationProfileMeaning {
+    readonly category?: string;
+    readonly information: string;
+    readonly name: string;
+    readonly priority: DisambiguationMeaningPriority;
+    readonly qid: string;
+}
+type DisambiguationProfileNormalizer = (input: DisambiguationProfileNormalizerInput) => Promise<DisambiguationProfile>;
+interface DisambiguationProfileNormalizerInput {
+    readonly pageQidLinks: readonly DisambiguationLinkedQid[];
+    readonly pages: readonly DisambiguationPageText[];
+    readonly sourceQid: string;
+    readonly surface?: string;
+}
+
+export type { CachedDisambiguationRecord, CachedPageRecord, CachedQidRecord, DisambiguationExpansion, DisambiguationLinkedQid, DisambiguationMeaningPriority, DisambiguationPageText, DisambiguationProfile, DisambiguationProfileError, DisambiguationProfileMeaning, DisambiguationProfileNormalizer, DisambiguationProfileNormalizerInput, EnrichmentStore, QidResolution, WikiClient, WikipageResolveProgress, WikipageResolveProgressDetail, WikipageResolveProgressReporter, WikipageResolverOptions, WikipageSitelink };

package/dist/external/wikipage/wikimedia-client/core.d.ts

@@ -0,0 +1,13 @@
+import { WikiClient } from '../types.js';
+import { WikimediaClientOptions, WikidataEntityInfo, SupportedWiki, WikiPageInfo, ParsedDisambiguationPage } from './wiki.js';
+import '../fetch-log.js';
+
+declare class WikimediaClient implements WikiClient {
+    #private;
+    constructor(options: WikimediaClientOptions);
+    getEntities(qids: readonly string[]): Promise<ReadonlyMap<string, WikidataEntityInfo>>;
+    getPagesByTitles(titles: readonly string[], wiki?: SupportedWiki): Promise<ReadonlyMap<string, WikiPageInfo>>;
+    parseDisambiguationPage(title: string, wiki: SupportedWiki): Promise<ParsedDisambiguationPage>;
+}
+
+export { WikimediaClient };

package/dist/external/wikipage/wikimedia-client/core.js

@@ -0,0 +1,207 @@
+import { getLogger } from "../../../runtime/common/logging.js";
+import { formatError } from "../../../utils/node-error.js";
+import { RateLimiter, parseRetryAfterMs } from "../rate-limiter.js";
+import { renderDisambiguationHtml } from "./html.js";
+import {
+  asArray,
+  asRecord,
+  getNestedString,
+  getNumber,
+  getString
+} from "./json.js";
+import {
+  delay,
+  getRetryDelayMs,
+  isRetryableError,
+  isTextResponse,
+  WikimediaRequestError
+} from "./request.js";
+import {
+  listWikidataLanguages,
+  normalizeWiki,
+  pickLocalizedValue,
+  SUPPORTED_WIKIS,
+  wikiApiBaseURL
+} from "./wiki.js";
+class WikimediaClient {
+  #fetch;
+  #language;
+  #limiter;
+  #requestLog;
+  #retryBaseDelayMs;
+  #retryTimes;
+  #userAgent;
+  #wiki;
+  constructor(options) {
+    this.#fetch = options.fetch ?? fetch;
+    this.#language = options.language;
+    this.#requestLog = options.requestLog;
+    this.#wiki = options.wiki;
+    this.#userAgent = options.userAgent;
+    this.#retryBaseDelayMs = Math.max(0, Math.floor(options.retryBaseDelayMs));
+    this.#retryTimes = Math.max(0, Math.floor(options.retryTimes));
+    this.#limiter = new RateLimiter({
+      concurrency: options.concurrency,
+      minRequestIntervalMs: options.minRequestIntervalMs
+    });
+  }
+  async getEntities(qids) {
+    if (qids.length === 0) {
+      return /* @__PURE__ */ new Map();
+    }
+    const url = new URL("https://www.wikidata.org/w/api.php");
+    url.searchParams.set("action", "wbgetentities");
+    url.searchParams.set("ids", qids.join("|"));
+    url.searchParams.set("props", "labels|descriptions|sitelinks");
+    url.searchParams.set("languages", listWikidataLanguages(this.#language));
+    url.searchParams.set("sitefilter", SUPPORTED_WIKIS.join("|"));
+    url.searchParams.set("format", "json");
+    url.searchParams.set("formatversion", "2");
+    const json = await this.#fetchJson(url);
+    const entities = asRecord(json.entities);
+    const results = /* @__PURE__ */ new Map();
+    for (const qid of qids) {
+      const entity = asRecord(entities[qid]);
+      const labels = asRecord(entity.labels);
+      const descriptions = asRecord(entity.descriptions);
+      results.set(qid, {
+        ...pickLocalizedValue(labels, this.#language) === void 0 ? {} : { label: pickLocalizedValue(labels, this.#language) },
+        ...pickLocalizedValue(descriptions, this.#language) === void 0 ? {} : { description: pickLocalizedValue(descriptions, this.#language) },
+        qid,
+        sitelinks: SUPPORTED_WIKIS.flatMap((wiki) => {
+          const title = getNestedString(entity, ["sitelinks", wiki, "title"]);
+          return title === void 0 ? [] : [{ title, wiki }];
+        })
+      });
+    }
+    return results;
+  }
+  async getPagesByTitles(titles, wiki = normalizeWiki(this.#wiki) ?? "enwiki") {
+    if (titles.length === 0) {
+      return /* @__PURE__ */ new Map();
+    }
+    const url = new URL(`${wikiApiBaseURL(wiki)}w/api.php`);
+    url.searchParams.set("action", "query");
+    url.searchParams.set("titles", titles.join("|"));
+    url.searchParams.set("prop", "pageprops");
+    url.searchParams.set("ppprop", "disambiguation|wikibase_item");
+    url.searchParams.set("format", "json");
+    url.searchParams.set("formatversion", "2");
+    const json = await this.#fetchJson(url);
+    const pages = asArray(asRecord(asRecord(json.query).pages));
+    const results = /* @__PURE__ */ new Map();
+    for (const pageValue of pages) {
+      const page = asRecord(pageValue);
+      const title = getString(page.title);
+      if (title === void 0) {
+        continue;
+      }
+      results.set(title, {
+        isDisambiguation: asRecord(page.pageprops).disambiguation !== void 0,
+        ...getNumber(page.pageid) === void 0 ? {} : { pageId: getNumber(page.pageid) },
+        title,
+        wiki,
+        ...getString(asRecord(page.pageprops).wikibase_item) === void 0 ? {} : {
+          wikibaseItem: getString(asRecord(page.pageprops).wikibase_item)
+        }
+      });
+    }
+    return results;
+  }
+  async parseDisambiguationPage(title, wiki) {
+    const url = new URL(`${wikiApiBaseURL(wiki)}w/api.php`);
+    url.searchParams.set("action", "parse");
+    url.searchParams.set("page", title);
+    url.searchParams.set("prop", "text|properties");
+    url.searchParams.set("format", "json");
+    url.searchParams.set("formatversion", "2");
+    const json = await this.#fetchJson(url);
+    const parse = asRecord(json.parse);
+    const rendered = renderDisambiguationHtml(getNestedString(parse, ["text"]));
+    return {
+      linkedTitles: rendered.linkedTitles,
+      ...getNumber(parse.pageid) === void 0 ? {} : { pageId: getNumber(parse.pageid) },
+      text: rendered.text,
+      title: getString(parse.title) ?? title,
+      wiki
+    };
+  }
+  async #fetchJson(url) {
+    let lastError;
+    for (let attempt = 0; attempt <= this.#retryTimes; attempt += 1) {
+      try {
+        return await this.#limiter.use(
+          async () => await this.#fetchJsonOnce(url, attempt + 1)
+        );
+      } catch (error) {
+        lastError = error;
+        if (!isRetryableError(error) || attempt >= this.#retryTimes) {
+          this.#requestLog.warnFailed();
+          throw error;
+        }
+        await delay(getRetryDelayMs(error, attempt, this.#retryBaseDelayMs));
+      }
+    }
+    throw lastError instanceof Error ? lastError : new Error(String(lastError));
+  }
+  async #fetchJsonOnce(url, attempt) {
+    const startedAt = Date.now();
+    let response;
+    try {
+      response = await this.#fetch(
+        url,
+        this.#userAgent === void 0 ? void 0 : {
+          headers: {
+            "User-Agent": this.#userAgent
+          }
+        }
+      );
+    } catch (error) {
+      await this.#appendFetchLog({
+        attempt,
+        durationMs: Date.now() - startedAt,
+        error,
+        startedAt,
+        url
+      });
+      throw error;
+    }
+    const retryAfterMs = parseRetryAfterMs(response.headers.get("retry-after"));
+    if (retryAfterMs !== void 0) {
+      this.#limiter.blockFor(retryAfterMs);
+    }
+    if (!response.ok) {
+      const responseText = isTextResponse(response) ? await response.text() : void 0;
+      await this.#appendFetchLog({
+        attempt,
+        durationMs: Date.now() - startedAt,
+        response,
+        ...responseText === void 0 ? {} : { responseText },
+        startedAt,
+        url
+      });
+      throw new WikimediaRequestError(url, response.status, retryAfterMs);
+    }
+    await this.#appendFetchLog({
+      attempt,
+      durationMs: Date.now() - startedAt,
+      response,
+      startedAt,
+      url
+    });
+    return asRecord(await response.json());
+  }
+  async #appendFetchLog(entry) {
+    try {
+      await this.#requestLog.append(entry);
+    } catch (error) {
+      getLogger({ component: "wikipage" }).warn(
+        `Failed to write wikipage fetch log entry: ${formatError(error)}`
+      );
+    }
+  }
+}
+export {
+  WikimediaClient
+};
+//# sourceMappingURL=core.js.map

package/dist/external/wikipage/wikimedia-client/core.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/external/wikipage/wikimedia-client/core.ts"],"sourcesContent":["import { getLogger } from \"../../../runtime/common/logging.js\";\nimport { formatError } from \"../../../utils/node-error.js\";\nimport type { WikipageFetchLog } from \"../fetch-log.js\";\nimport type { WikiClient } from \"../types.js\";\nimport { RateLimiter, parseRetryAfterMs } from \"../rate-limiter.js\";\nimport { renderDisambiguationHtml } from \"./html.js\";\nimport {\n  asArray,\n  asRecord,\n  getNestedString,\n  getNumber,\n  getString,\n} from \"./json.js\";\nimport {\n  delay,\n  getRetryDelayMs,\n  isRetryableError,\n  isTextResponse,\n  WikimediaRequestError,\n} from \"./request.js\";\nimport {\n  listWikidataLanguages,\n  type MediaWikiPage,\n  normalizeWiki,\n  pickLocalizedValue,\n  SUPPORTED_WIKIS,\n  type ParsedDisambiguationPage,\n  type SupportedWiki,\n  type WikidataEntityInfo,\n  type WikimediaClientOptions,\n  type WikiPageInfo,\n  wikiApiBaseURL,\n} from \"./wiki.js\";\n\nexport class WikimediaClient implements WikiClient {\n  readonly #fetch: typeof fetch;\n  readonly #language: string;\n  readonly #limiter: RateLimiter;\n  readonly #requestLog: WikipageFetchLog;\n  readonly #retryBaseDelayMs: number;\n  readonly #retryTimes: number;\n  readonly #userAgent: string | undefined;\n  readonly #wiki: string;\n\n  public constructor(options: WikimediaClientOptions) {\n    this.#fetch = options.fetch ?? fetch;\n    this.#language = options.language;\n    this.#requestLog = options.requestLog;\n    this.#wiki = options.wiki;\n    this.#userAgent = options.userAgent;\n    this.#retryBaseDelayMs = Math.max(0, Math.floor(options.retryBaseDelayMs));\n    this.#retryTimes = Math.max(0, Math.floor(options.retryTimes));\n    this.#limiter = new RateLimiter({\n      concurrency: options.concurrency,\n      minRequestIntervalMs: options.minRequestIntervalMs,\n    });\n  }\n\n  public async getEntities(\n    qids: readonly string[],\n  ): Promise<ReadonlyMap<string, WikidataEntityInfo>> {\n    if (qids.length === 0) {\n      return new Map();\n    }\n\n    const url = new URL(\"https://www.wikidata.org/w/api.php\");\n    url.searchParams.set(\"action\", \"wbgetentities\");\n    url.searchParams.set(\"ids\", qids.join(\"|\"));\n    url.searchParams.set(\"props\", \"labels|descriptions|sitelinks\");\n    url.searchParams.set(\"languages\", listWikidataLanguages(this.#language));\n    url.searchParams.set(\"sitefilter\", SUPPORTED_WIKIS.join(\"|\"));\n    url.searchParams.set(\"format\", \"json\");\n    url.searchParams.set(\"formatversion\", \"2\");\n\n    const json = await this.#fetchJson(url);\n    const entities = asRecord(json.entities);\n    const results = new Map<string, WikidataEntityInfo>();\n\n    for (const qid of qids) {\n      const entity = asRecord(entities[qid]);\n      const labels = asRecord(entity.labels);\n      const descriptions = asRecord(entity.descriptions);\n\n      results.set(qid, {\n        ...(pickLocalizedValue(labels, this.#language) === undefined\n          ? {}\n          : { label: pickLocalizedValue(labels, this.#language)! }),\n        ...(pickLocalizedValue(descriptions, this.#language) === undefined\n          ? {}\n          : { description: pickLocalizedValue(descriptions, this.#language)! }),\n        qid,\n        sitelinks: SUPPORTED_WIKIS.flatMap((wiki) => {\n          const title = getNestedString(entity, [\"sitelinks\", wiki, \"title\"]);\n\n          return title === undefined ? [] : [{ title, wiki }];\n        }),\n      });\n    }\n\n    return results;\n  }\n\n  public async getPagesByTitles(\n    titles: readonly string[],\n    wiki: SupportedWiki = normalizeWiki(this.#wiki) ?? \"enwiki\",\n  ): Promise<ReadonlyMap<string, WikiPageInfo>> {\n    if (titles.length === 0) {\n      return new Map();\n    }\n\n    const url = new URL(`${wikiApiBaseURL(wiki)}w/api.php`);\n    url.searchParams.set(\"action\", \"query\");\n    url.searchParams.set(\"titles\", titles.join(\"|\"));\n    url.searchParams.set(\"prop\", \"pageprops\");\n    url.searchParams.set(\"ppprop\", \"disambiguation|wikibase_item\");\n    url.searchParams.set(\"format\", \"json\");\n    url.searchParams.set(\"formatversion\", \"2\");\n\n    const json = await this.#fetchJson(url);\n    const pages = asArray(asRecord(asRecord(json.query).pages));\n    const results = new Map<string, WikiPageInfo>();\n\n    for (const pageValue of pages) {\n      const page = asRecord(pageValue) as MediaWikiPage;\n      const title = getString(page.title);\n\n      if (title === undefined) {\n        continue;\n      }\n\n      results.set(title, {\n        isDisambiguation: asRecord(page.pageprops).disambiguation !== undefined,\n        ...(getNumber(page.pageid) === undefined\n          ? {}\n          : { pageId: getNumber(page.pageid)! }),\n        title,\n        wiki,\n        ...(getString(asRecord(page.pageprops).wikibase_item) === undefined\n          ? {}\n          : {\n              wikibaseItem: getString(asRecord(page.pageprops).wikibase_item)!,\n            }),\n      });\n    }\n\n    return results;\n  }\n\n  public async parseDisambiguationPage(\n    title: string,\n    wiki: SupportedWiki,\n  ): Promise<ParsedDisambiguationPage> {\n    const url = new URL(`${wikiApiBaseURL(wiki)}w/api.php`);\n    url.searchParams.set(\"action\", \"parse\");\n    url.searchParams.set(\"page\", title);\n    url.searchParams.set(\"prop\", \"text|properties\");\n    url.searchParams.set(\"format\", \"json\");\n    url.searchParams.set(\"formatversion\", \"2\");\n\n    const json = await this.#fetchJson(url);\n    const parse = asRecord(json.parse);\n    const rendered = renderDisambiguationHtml(getNestedString(parse, [\"text\"]));\n\n    return {\n      linkedTitles: rendered.linkedTitles,\n      ...(getNumber(parse.pageid) === undefined\n        ? {}\n        : { pageId: getNumber(parse.pageid)! }),\n      text: rendered.text,\n      title: getString(parse.title) ?? title,\n      wiki,\n    };\n  }\n\n  async #fetchJson(url: URL): Promise<Record<string, unknown>> {\n    let lastError: unknown;\n\n    for (let attempt = 0; attempt <= this.#retryTimes; attempt += 1) {\n      try {\n        return await this.#limiter.use(\n          async () => await this.#fetchJsonOnce(url, attempt + 1),\n        );\n      } catch (error) {\n        lastError = error;\n\n        if (!isRetryableError(error) || attempt >= this.#retryTimes) {\n          this.#requestLog.warnFailed();\n          throw error;\n        }\n\n        await delay(getRetryDelayMs(error, attempt, this.#retryBaseDelayMs));\n      }\n    }\n\n    throw lastError instanceof Error ? lastError : new Error(String(lastError));\n  }\n\n  async #fetchJsonOnce(\n    url: URL,\n    attempt: number,\n  ): Promise<Record<string, unknown>> {\n    const startedAt = Date.now();\n    let response: Response;\n\n    try {\n      response = await this.#fetch(\n        url,\n        this.#userAgent === undefined\n          ? undefined\n          : {\n              headers: {\n                \"User-Agent\": this.#userAgent,\n              },\n            },\n      );\n    } catch (error) {\n      await this.#appendFetchLog({\n        attempt,\n        durationMs: Date.now() - startedAt,\n        error,\n        startedAt,\n        url,\n      });\n      throw error;\n    }\n\n    const retryAfterMs = parseRetryAfterMs(response.headers.get(\"retry-after\"));\n\n    if (retryAfterMs !== undefined) {\n      this.#limiter.blockFor(retryAfterMs);\n    }\n    if (!response.ok) {\n      const responseText = isTextResponse(response)\n        ? await response.text()\n        : undefined;\n\n      await this.#appendFetchLog({\n        attempt,\n        durationMs: Date.now() - startedAt,\n        response,\n        ...(responseText === undefined ? {} : { responseText }),\n        startedAt,\n        url,\n      });\n      throw new WikimediaRequestError(url, response.status, retryAfterMs);\n    }\n\n    await this.#appendFetchLog({\n      attempt,\n      durationMs: Date.now() - startedAt,\n      response,\n      startedAt,\n      url,\n    });\n\n    return asRecord(await response.json());\n  }\n\n  async #appendFetchLog(\n    entry: Parameters<WikipageFetchLog[\"append\"]>[0],\n  ): Promise<void> {\n    try {\n      await this.#requestLog.append(entry);\n    } catch (error) {\n      getLogger({ component: \"wikipage\" }).warn(\n        `Failed to write wikipage fetch log entry: ${formatError(error)}`,\n      );\n    }\n  }\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB;AAG5B,SAAS,aAAa,yBAAyB;AAC/C,SAAS,gCAAgC;AACzC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EAMA;AAAA,OACK;AAEA,MAAM,gBAAsC;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEF,YAAY,SAAiC;AAClD,SAAK,SAAS,QAAQ,SAAS;AAC/B,SAAK,YAAY,QAAQ;AACzB,SAAK,cAAc,QAAQ;AAC3B,SAAK,QAAQ,QAAQ;AACrB,SAAK,aAAa,QAAQ;AAC1B,SAAK,oBAAoB,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,gBAAgB,CAAC;AACzE,SAAK,cAAc,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,UAAU,CAAC;AAC7D,SAAK,WAAW,IAAI,YAAY;AAAA,MAC9B,aAAa,QAAQ;AAAA,MACrB,sBAAsB,QAAQ;AAAA,IAChC,CAAC;AAAA,EACH;AAAA,EAEA,MAAa,YACX,MACkD;AAClD,QAAI,KAAK,WAAW,GAAG;AACrB,aAAO,oBAAI,IAAI;AAAA,IACjB;AAEA,UAAM,MAAM,IAAI,IAAI,oCAAoC;AACxD,QAAI,aAAa,IAAI,UAAU,eAAe;AAC9C,QAAI,aAAa,IAAI,OAAO,KAAK,KAAK,GAAG,CAAC;AAC1C,QAAI,aAAa,IAAI,SAAS,+BAA+B;AAC7D,QAAI,aAAa,IAAI,aAAa,sBAAsB,KAAK,SAAS,CAAC;AACvE,QAAI,aAAa,IAAI,cAAc,gBAAgB,KAAK,GAAG,CAAC;AAC5D,QAAI,aAAa,IAAI,UAAU,MAAM;AACrC,QAAI,aAAa,IAAI,iBAAiB,GAAG;AAEzC,UAAM,OAAO,MAAM,KAAK,WAAW,GAAG;AACtC,UAAM,WAAW,SAAS,KAAK,QAAQ;AACvC,UAAM,UAAU,oBAAI,IAAgC;AAEpD,eAAW,OAAO,MAAM;AACtB,YAAM,SAAS,SAAS,SAAS,GAAG,CAAC;AACrC,YAAM,SAAS,SAAS,OAAO,MAAM;AACrC,YAAM,eAAe,SAAS,OAAO,YAAY;AAEjD,cAAQ,IAAI,KAAK;AAAA,QACf,GAAI,mBAAmB,QAAQ,KAAK,SAAS,MAAM,SAC/C,CAAC,IACD,EAAE,OAAO,mBAAmB,QAAQ,KAAK,SAAS,EAAG;AAAA,QACzD,GAAI,mBAAmB,cAAc,KAAK,SAAS,MAAM,SACrD,CAAC,IACD,EAAE,aAAa,mBAAmB,cAAc,KAAK,SAAS,EAAG;AAAA,QACrE;AAAA,QACA,WAAW,gBAAgB,QAAQ,CAAC,SAAS;AAC3C,gBAAM,QAAQ,gBAAgB,QAAQ,CAAC,aAAa,MAAM,OAAO,CAAC;AAElE,iBAAO,UAAU,SAAY,CAAC,IAAI,CAAC,EAAE,OAAO,KAAK,CAAC;AAAA,QACpD,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAa,iBACX,QACA,OAAsB,cAAc,KAAK,KAAK,KAAK,UACP;AAC5C,QAAI,OAAO,WAAW,GAAG;AACvB,aAAO,oBAAI,IAAI;AAAA,IACjB;AAEA,UAAM,MAAM,IAAI,IAAI,GAAG,eAAe,IAAI,CAAC,WAAW;AACtD,QAAI,aAAa,IAAI,UAAU,OAAO;AACtC,QAAI,aAAa,IAAI,UAAU,OAAO,KAAK,GAAG,CAAC;AAC/C,QAAI,aAAa,IAAI,QAAQ,WAAW;AACxC,QAAI,aAAa,IAAI,UAAU,8BAA8B;AAC7D,QAAI,aAAa,IAAI,UAAU,MAAM;AACrC,QAAI,aAAa,IAAI,iBAAiB,GAAG;AAEzC,UAAM,OAAO,MAAM,KAAK,WAAW,GAAG;AACtC,UAAM,QAAQ,QAAQ,SAAS,SAAS,KAAK,KAAK,EAAE,KAAK,CAAC;AAC1D,UAAM,UAAU,oBAAI,IAA0B;AAE9C,eAAW,aAAa,OAAO;AAC7B,YAAM,OAAO,SAAS,SAAS;AAC/B,YAAM,QAAQ,UAAU,KAAK,KAAK;AAElC,UAAI,UAAU,QAAW;AACvB;AAAA,MACF;AAEA,cAAQ,IAAI,OAAO;AAAA,QACjB,kBAAkB,SAAS,KAAK,SAAS,EAAE,mBAAmB;AAAA,QAC9D,GAAI,UAAU,KAAK,MAAM,MAAM,SAC3B,CAAC,IACD,EAAE,QAAQ,UAAU,KAAK,MAAM,EAAG;AAAA,QACtC;AAAA,QACA;AAAA,QACA,GAAI,UAAU,SAAS,KAAK,SAAS,EAAE,aAAa,MAAM,SACtD,CAAC,IACD;AAAA,UACE,cAAc,UAAU,SAAS,KAAK,SAAS,EAAE,aAAa;AAAA,QAChE;AAAA,MACN,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAa,wBACX,OACA,MACmC;AACnC,UAAM,MAAM,IAAI,IAAI,GAAG,eAAe,IAAI,CAAC,WAAW;AACtD,QAAI,aAAa,IAAI,UAAU,OAAO;AACtC,QAAI,aAAa,IAAI,QAAQ,KAAK;AAClC,QAAI,aAAa,IAAI,QAAQ,iBAAiB;AAC9C,QAAI,aAAa,IAAI,UAAU,MAAM;AACrC,QAAI,aAAa,IAAI,iBAAiB,GAAG;AAEzC,UAAM,OAAO,MAAM,KAAK,WAAW,GAAG;AACtC,UAAM,QAAQ,SAAS,KAAK,KAAK;AACjC,UAAM,WAAW,yBAAyB,gBAAgB,OAAO,CAAC,MAAM,CAAC,CAAC;AAE1E,WAAO;AAAA,MACL,cAAc,SAAS;AAAA,MACvB,GAAI,UAAU,MAAM,MAAM,MAAM,SAC5B,CAAC,IACD,EAAE,QAAQ,UAAU,MAAM,MAAM,EAAG;AAAA,MACvC,MAAM,SAAS;AAAA,MACf,OAAO,UAAU,MAAM,KAAK,KAAK;AAAA,MACjC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAW,KAA4C;AAC3D,QAAI;AAEJ,aAAS,UAAU,GAAG,WAAW,KAAK,aAAa,WAAW,GAAG;AAC/D,UAAI;AACF,eAAO,MAAM,KAAK,SAAS;AAAA,UACzB,YAAY,MAAM,KAAK,eAAe,KAAK,UAAU,CAAC;AAAA,QACxD;AAAA,MACF,SAAS,OAAO;AACd,oBAAY;AAEZ,YAAI,CAAC,iBAAiB,KAAK,KAAK,WAAW,KAAK,aAAa;AAC3D,eAAK,YAAY,WAAW;AAC5B,gBAAM;AAAA,QACR;AAEA,cAAM,MAAM,gBAAgB,OAAO,SAAS,KAAK,iBAAiB,CAAC;AAAA,MACrE;AAAA,IACF;AAEA,UAAM,qBAAqB,QAAQ,YAAY,IAAI,MAAM,OAAO,SAAS,CAAC;AAAA,EAC5E;AAAA,EAEA,MAAM,eACJ,KACA,SACkC;AAClC,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AAEJ,QAAI;AACF,iBAAW,MAAM,KAAK;AAAA,QACpB;AAAA,QACA,KAAK,eAAe,SAChB,SACA;AAAA,UACE,SAAS;AAAA,YACP,cAAc,KAAK;AAAA,UACrB;AAAA,QACF;AAAA,MACN;AAAA,IACF,SAAS,OAAO;AACd,YAAM,KAAK,gBAAgB;AAAA,QACzB;AAAA,QACA,YAAY,KAAK,IAAI,IAAI;AAAA,QACzB;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AACD,YAAM;AAAA,IACR;AAEA,UAAM,eAAe,kBAAkB,SAAS,QAAQ,IAAI,aAAa,CAAC;AAE1E,QAAI,iBAAiB,QAAW;AAC9B,WAAK,SAAS,SAAS,YAAY;AAAA,IACrC;AACA,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,eAAe,eAAe,QAAQ,IACxC,MAAM,SAAS,KAAK,IACpB;AAEJ,YAAM,KAAK,gBAAgB;AAAA,QACzB;AAAA,QACA,YAAY,KAAK,IAAI,IAAI;AAAA,QACzB;AAAA,QACA,GAAI,iBAAiB,SAAY,CAAC,IAAI,EAAE,aAAa;AAAA,QACrD;AAAA,QACA;AAAA,MACF,CAAC;AACD,YAAM,IAAI,sBAAsB,KAAK,SAAS,QAAQ,YAAY;AAAA,IACpE;AAEA,UAAM,KAAK,gBAAgB;AAAA,MACzB;AAAA,MACA,YAAY,KAAK,IAAI,IAAI;AAAA,MACzB;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAED,WAAO,SAAS,MAAM,SAAS,KAAK,CAAC;AAAA,EACvC;AAAA,EAEA,MAAM,gBACJ,OACe;AACf,QAAI;AACF,YAAM,KAAK,YAAY,OAAO,KAAK;AAAA,IACrC,SAAS,OAAO;AACd,gBAAU,EAAE,WAAW,WAAW,CAAC,EAAE;AAAA,QACnC,6CAA6C,YAAY,KAAK,CAAC;AAAA,MACjE;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/external/wikipage/wikimedia-client/html.d.ts

@@ -0,0 +1,7 @@
+declare function replaceTitleUriWithQidUri(text: string, titleToQid: ReadonlyMap<string, string>): string;
+declare function renderDisambiguationHtml(html: string | undefined): {
+    readonly linkedTitles: readonly string[];
+    readonly text: string;
+};
+
+export { renderDisambiguationHtml, replaceTitleUriWithQidUri };

package/dist/external/wikipage/wikimedia-client/html.js

@@ -0,0 +1,136 @@
+import { DomUtils, parseDocument } from "htmlparser2";
+import { asRecord, getString } from "./json.js";
+const TITLE_URI_PREFIX = "wikigraph-title://";
+function replaceTitleUriWithQidUri(text, titleToQid) {
+  return text.replace(
+    /\[\[([^\]|]+)\|wikigraph-title:\/\/([^\]\s]+)\]\]/gu,
+    (_match, label, encodedTitle) => {
+      const title = decodeURIComponent(encodedTitle);
+      const qid = titleToQid.get(title);
+      return qid === void 0 ? label : `[[${label}|wikg://qid=${qid}]]`;
+    }
+  );
+}
+function renderDisambiguationHtml(html) {
+  if (html === void 0 || html.trim() === "") {
+    return {
+      linkedTitles: [],
+      text: ""
+    };
+  }
+  const linkedTitles = /* @__PURE__ */ new Set();
+  const document = parseDocument(html);
+  const text = renderNodes(DomUtils.getChildren(document), {
+    linkedTitles,
+    listDepth: 0
+  }).replace(/[ \t]+\n/gu, "\n").replace(/\n{3,}/gu, "\n\n").trim();
+  return {
+    linkedTitles: [...linkedTitles],
+    text
+  };
+}
+function renderNodes(nodes, context) {
+  return nodes.map((node) => renderNode(node, context)).join("");
+}
+function renderNode(node, context) {
+  const record = asRecord(node);
+  const nodeType = getString(record.type);
+  if (nodeType === "text") {
+    return normalizeInlineText(getString(record.data) ?? "");
+  }
+  if (nodeType !== "tag" && nodeType !== "script" && nodeType !== "style") {
+    return "";
+  }
+  const element = node;
+  if (shouldSkipElement(element)) {
+    return "";
+  }
+  const name = element.name.toLowerCase();
+  if (name === "a") {
+    return renderLink(element, context);
+  }
+  if (/^h[1-6]$/u.test(name)) {
+    const level = Math.min(6, Math.max(1, Number(name.slice(1))));
+    const marker = "=".repeat(level);
+    const heading = renderNodes(element.children ?? [], context).trim();
+    return heading === "" ? "" : `
+${marker} ${heading} ${marker}
+`;
+  }
+  if (name === "li") {
+    const indent = "  ".repeat(Math.max(0, context.listDepth - 1));
+    const content = renderNodes(element.children ?? [], {
+      ...context,
+      listDepth: context.listDepth + 1
+    }).trim();
+    return content === "" ? "" : `
+${indent}* ${content}`;
+  }
+  if (name === "ul" || name === "ol") {
+    return `${renderNodes(element.children ?? [], context)}
+`;
+  }
+  if (isBlockElement(name)) {
+    const content = renderNodes(element.children ?? [], context).trim();
+    return content === "" ? "" : `
+${content}
+`;
+  }
+  return renderNodes(element.children ?? [], context);
+}
+function renderLink(node, context) {
+  const content = renderNodes(node.children ?? [], context).trim();
+  const title = extractWikiLinkTitle(node);
+  if (title === void 0) {
+    return content;
+  }
+  context.linkedTitles.add(title);
+  return `[[${content === "" ? title : content}|${TITLE_URI_PREFIX}${encodeURIComponent(title)}]]`;
+}
+function extractWikiLinkTitle(node) {
+  const title = node.attribs.title;
+  const href = node.attribs.href;
+  if (title !== void 0 && title !== "" && !title.includes(":")) {
+    return title;
+  }
+  if (href === void 0 || !href.startsWith("/wiki/")) {
+    return void 0;
+  }
+  const decoded = decodeURIComponent(href.slice("/wiki/".length)).replaceAll(
+    "_",
+    " "
+  );
+  return decoded === "" || decoded.includes(":") ? void 0 : decoded;
+}
+function shouldSkipElement(node) {
+  const name = node.name.toLowerCase();
+  if (["script", "style", "table", "sup"].includes(name)) {
+    return true;
+  }
+  const className = node.attribs.class ?? "";
+  return [
+    "catlinks",
+    "metadata",
+    "mw-editsection",
+    "mw-empty-elt",
+    "navbox",
+    "noprint",
+    "reference",
+    "reflist",
+    "shortdescription",
+    "toc"
+  ].some((item) => className.split(/\s+/u).includes(item));
+}
+function isBlockElement(name) {
+  return ["blockquote", "br", "dd", "div", "dl", "dt", "p", "section"].includes(
+    name
+  );
+}
+function normalizeInlineText(value) {
+  return value.replace(/\s+/gu, " ");
+}
+export {
+  renderDisambiguationHtml,
+  replaceTitleUriWithQidUri
+};
+//# sourceMappingURL=html.js.map

package/dist/external/wikipage/wikimedia-client/html.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/external/wikipage/wikimedia-client/html.ts"],"sourcesContent":["import { DomUtils, parseDocument } from \"htmlparser2\";\n\nimport { asRecord, getString } from \"./json.js\";\n\nconst TITLE_URI_PREFIX = \"wikigraph-title://\";\n\ninterface HtmlElement {\n  readonly attribs: Record<string, string | undefined>;\n  readonly children?: readonly unknown[];\n  readonly name: string;\n}\n\nexport function replaceTitleUriWithQidUri(\n  text: string,\n  titleToQid: ReadonlyMap<string, string>,\n): string {\n  return text.replace(\n    /\\[\\[([^\\]|]+)\\|wikigraph-title:\\/\\/([^\\]\\s]+)\\]\\]/gu,\n    (_match: string, label: string, encodedTitle: string) => {\n      const title = decodeURIComponent(encodedTitle);\n      const qid = titleToQid.get(title);\n\n      return qid === undefined ? label : `[[${label}|wikg://qid=${qid}]]`;\n    },\n  );\n}\n\nexport function renderDisambiguationHtml(html: string | undefined): {\n  readonly linkedTitles: readonly string[];\n  readonly text: string;\n} {\n  if (html === undefined || html.trim() === \"\") {\n    return {\n      linkedTitles: [],\n      text: \"\",\n    };\n  }\n\n  const linkedTitles = new Set<string>();\n  const document = parseDocument(html);\n  const text = renderNodes(DomUtils.getChildren(document), {\n    linkedTitles,\n    listDepth: 0,\n  })\n    .replace(/[ \\t]+\\n/gu, \"\\n\")\n    .replace(/\\n{3,}/gu, \"\\n\\n\")\n    .trim();\n\n  return {\n    linkedTitles: [...linkedTitles],\n    text,\n  };\n}\n\nfunction renderNodes(\n  nodes: readonly unknown[],\n  context: {\n    readonly linkedTitles: Set<string>;\n    readonly listDepth: number;\n  },\n): string {\n  return nodes.map((node) => renderNode(node, context)).join(\"\");\n}\n\nfunction renderNode(\n  node: unknown,\n  context: {\n    readonly linkedTitles: Set<string>;\n    readonly listDepth: number;\n  },\n): string {\n  const record = asRecord(node);\n  const nodeType = getString(record.type);\n\n  if (nodeType === \"text\") {\n    return normalizeInlineText(getString(record.data) ?? \"\");\n  }\n  if (nodeType !== \"tag\" && nodeType !== \"script\" && nodeType !== \"style\") {\n    return \"\";\n  }\n\n  const element = node as HtmlElement;\n  if (shouldSkipElement(element)) {\n    return \"\";\n  }\n\n  const name = element.name.toLowerCase();\n\n  if (name === \"a\") {\n    return renderLink(element, context);\n  }\n  if (/^h[1-6]$/u.test(name)) {\n    const level = Math.min(6, Math.max(1, Number(name.slice(1))));\n    const marker = \"=\".repeat(level);\n    const heading = renderNodes(element.children ?? [], context).trim();\n\n    return heading === \"\" ? \"\" : `\\n${marker} ${heading} ${marker}\\n`;\n  }\n  if (name === \"li\") {\n    const indent = \"  \".repeat(Math.max(0, context.listDepth - 1));\n    const content = renderNodes(element.children ?? [], {\n      ...context,\n      listDepth: context.listDepth + 1,\n    }).trim();\n\n    return content === \"\" ? \"\" : `\\n${indent}* ${content}`;\n  }\n  if (name === \"ul\" || name === \"ol\") {\n    return `${renderNodes(element.children ?? [], context)}\\n`;\n  }\n  if (isBlockElement(name)) {\n    const content = renderNodes(element.children ?? [], context).trim();\n\n    return content === \"\" ? \"\" : `\\n${content}\\n`;\n  }\n\n  return renderNodes(element.children ?? [], context);\n}\n\nfunction renderLink(\n  node: HtmlElement,\n  context: {\n    readonly linkedTitles: Set<string>;\n    readonly listDepth: number;\n  },\n): string {\n  const content = renderNodes(node.children ?? [], context).trim();\n  const title = extractWikiLinkTitle(node);\n\n  if (title === undefined) {\n    return content;\n  }\n\n  context.linkedTitles.add(title);\n\n  return `[[${content === \"\" ? title : content}|${TITLE_URI_PREFIX}${encodeURIComponent(title)}]]`;\n}\n\nfunction extractWikiLinkTitle(node: HtmlElement): string | undefined {\n  const title = node.attribs.title;\n  const href = node.attribs.href;\n\n  if (title !== undefined && title !== \"\" && !title.includes(\":\")) {\n    return title;\n  }\n  if (href === undefined || !href.startsWith(\"/wiki/\")) {\n    return undefined;\n  }\n\n  const decoded = decodeURIComponent(href.slice(\"/wiki/\".length)).replaceAll(\n    \"_\",\n    \" \",\n  );\n\n  return decoded === \"\" || decoded.includes(\":\") ? undefined : decoded;\n}\n\nfunction shouldSkipElement(node: HtmlElement): boolean {\n  const name = node.name.toLowerCase();\n\n  if ([\"script\", \"style\", \"table\", \"sup\"].includes(name)) {\n    return true;\n  }\n\n  const className = node.attribs.class ?? \"\";\n\n  return [\n    \"catlinks\",\n    \"metadata\",\n    \"mw-editsection\",\n    \"mw-empty-elt\",\n    \"navbox\",\n    \"noprint\",\n    \"reference\",\n    \"reflist\",\n    \"shortdescription\",\n    \"toc\",\n  ].some((item) => className.split(/\\s+/u).includes(item));\n}\n\nfunction isBlockElement(name: string): boolean {\n  return [\"blockquote\", \"br\", \"dd\", \"div\", \"dl\", \"dt\", \"p\", \"section\"].includes(\n    name,\n  );\n}\n\nfunction normalizeInlineText(value: string): string {\n  return value.replace(/\\s+/gu, \" \");\n}\n"],"mappings":"AAAA,SAAS,UAAU,qBAAqB;AAExC,SAAS,UAAU,iBAAiB;AAEpC,MAAM,mBAAmB;AAQlB,SAAS,0BACd,MACA,YACQ;AACR,SAAO,KAAK;AAAA,IACV;AAAA,IACA,CAAC,QAAgB,OAAe,iBAAyB;AACvD,YAAM,QAAQ,mBAAmB,YAAY;AAC7C,YAAM,MAAM,WAAW,IAAI,KAAK;AAEhC,aAAO,QAAQ,SAAY,QAAQ,KAAK,KAAK,eAAe,GAAG;AAAA,IACjE;AAAA,EACF;AACF;AAEO,SAAS,yBAAyB,MAGvC;AACA,MAAI,SAAS,UAAa,KAAK,KAAK,MAAM,IAAI;AAC5C,WAAO;AAAA,MACL,cAAc,CAAC;AAAA,MACf,MAAM;AAAA,IACR;AAAA,EACF;AAEA,QAAM,eAAe,oBAAI,IAAY;AACrC,QAAM,WAAW,cAAc,IAAI;AACnC,QAAM,OAAO,YAAY,SAAS,YAAY,QAAQ,GAAG;AAAA,IACvD;AAAA,IACA,WAAW;AAAA,EACb,CAAC,EACE,QAAQ,cAAc,IAAI,EAC1B,QAAQ,YAAY,MAAM,EAC1B,KAAK;AAER,SAAO;AAAA,IACL,cAAc,CAAC,GAAG,YAAY;AAAA,IAC9B;AAAA,EACF;AACF;AAEA,SAAS,YACP,OACA,SAIQ;AACR,SAAO,MAAM,IAAI,CAAC,SAAS,WAAW,MAAM,OAAO,CAAC,EAAE,KAAK,EAAE;AAC/D;AAEA,SAAS,WACP,MACA,SAIQ;AACR,QAAM,SAAS,SAAS,IAAI;AAC5B,QAAM,WAAW,UAAU,OAAO,IAAI;AAEtC,MAAI,aAAa,QAAQ;AACvB,WAAO,oBAAoB,UAAU,OAAO,IAAI,KAAK,EAAE;AAAA,EACzD;AACA,MAAI,aAAa,SAAS,aAAa,YAAY,aAAa,SAAS;AACvE,WAAO;AAAA,EACT;AAEA,QAAM,UAAU;AAChB,MAAI,kBAAkB,OAAO,GAAG;AAC9B,WAAO;AAAA,EACT;AAEA,QAAM,OAAO,QAAQ,KAAK,YAAY;AAEtC,MAAI,SAAS,KAAK;AAChB,WAAO,WAAW,SAAS,OAAO;AAAA,EACpC;AACA,MAAI,YAAY,KAAK,IAAI,GAAG;AAC1B,UAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC;AAC5D,UAAM,SAAS,IAAI,OAAO,KAAK;AAC/B,UAAM,UAAU,YAAY,QAAQ,YAAY,CAAC,GAAG,OAAO,EAAE,KAAK;AAElE,WAAO,YAAY,KAAK,KAAK;AAAA,EAAK,MAAM,IAAI,OAAO,IAAI,MAAM;AAAA;AAAA,EAC/D;AACA,MAAI,SAAS,MAAM;AACjB,UAAM,SAAS,KAAK,OAAO,KAAK,IAAI,GAAG,QAAQ,YAAY,CAAC,CAAC;AAC7D,UAAM,UAAU,YAAY,QAAQ,YAAY,CAAC,GAAG;AAAA,MAClD,GAAG;AAAA,MACH,WAAW,QAAQ,YAAY;AAAA,IACjC,CAAC,EAAE,KAAK;AAER,WAAO,YAAY,KAAK,KAAK;AAAA,EAAK,MAAM,KAAK,OAAO;AAAA,EACtD;AACA,MAAI,SAAS,QAAQ,SAAS,MAAM;AAClC,WAAO,GAAG,YAAY,QAAQ,YAAY,CAAC,GAAG,OAAO,CAAC;AAAA;AAAA,EACxD;AACA,MAAI,eAAe,IAAI,GAAG;AACxB,UAAM,UAAU,YAAY,QAAQ,YAAY,CAAC,GAAG,OAAO,EAAE,KAAK;AAElE,WAAO,YAAY,KAAK,KAAK;AAAA,EAAK,OAAO;AAAA;AAAA,EAC3C;AAEA,SAAO,YAAY,QAAQ,YAAY,CAAC,GAAG,OAAO;AACpD;AAEA,SAAS,WACP,MACA,SAIQ;AACR,QAAM,UAAU,YAAY,KAAK,YAAY,CAAC,GAAG,OAAO,EAAE,KAAK;AAC/D,QAAM,QAAQ,qBAAqB,IAAI;AAEvC,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AAEA,UAAQ,aAAa,IAAI,KAAK;AAE9B,SAAO,KAAK,YAAY,KAAK,QAAQ,OAAO,IAAI,gBAAgB,GAAG,mBAAmB,KAAK,CAAC;AAC9F;AAEA,SAAS,qBAAqB,MAAuC;AACnE,QAAM,QAAQ,KAAK,QAAQ;AAC3B,QAAM,OAAO,KAAK,QAAQ;AAE1B,MAAI,UAAU,UAAa,UAAU,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG;AAC/D,WAAO;AAAA,EACT;AACA,MAAI,SAAS,UAAa,CAAC,KAAK,WAAW,QAAQ,GAAG;AACpD,WAAO;AAAA,EACT;AAEA,QAAM,UAAU,mBAAmB,KAAK,MAAM,SAAS,MAAM,CAAC,EAAE;AAAA,IAC9D;AAAA,IACA;AAAA,EACF;AAEA,SAAO,YAAY,MAAM,QAAQ,SAAS,GAAG,IAAI,SAAY;AAC/D;AAEA,SAAS,kBAAkB,MAA4B;AACrD,QAAM,OAAO,KAAK,KAAK,YAAY;AAEnC,MAAI,CAAC,UAAU,SAAS,SAAS,KAAK,EAAE,SAAS,IAAI,GAAG;AACtD,WAAO;AAAA,EACT;AAEA,QAAM,YAAY,KAAK,QAAQ,SAAS;AAExC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,CAAC,SAAS,UAAU,MAAM,MAAM,EAAE,SAAS,IAAI,CAAC;AACzD;AAEA,SAAS,eAAe,MAAuB;AAC7C,SAAO,CAAC,cAAc,MAAM,MAAM,OAAO,MAAM,MAAM,KAAK,SAAS,EAAE;AAAA,IACnE;AAAA,EACF;AACF;AAEA,SAAS,oBAAoB,OAAuB;AAClD,SAAO,MAAM,QAAQ,SAAS,GAAG;AACnC;","names":[]}

package/dist/external/wikipage/wikimedia-client/index.d.ts

@@ -0,0 +1,5 @@
+export { WikimediaClient } from './core.js';
+export { renderDisambiguationHtml, replaceTitleUriWithQidUri } from './html.js';
+export { MediaWikiPage, ParsedDisambiguationPage, SUPPORTED_WIKIS, SupportedWiki, WikiPageInfo, WikidataEntityInfo, WikidataSitelinkInfo, WikimediaClientOptions, listWikidataLanguages, normalizeWiki, pickLocalizedValue, wikiApiBaseURL } from './wiki.js';
+import '../types.js';
+import '../fetch-log.js';

package/dist/external/wikipage/wikimedia-client/index.js

@@ -0,0 +1,4 @@
+export * from "./core.js";
+export * from "./html.js";
+export * from "./wiki.js";
+//# sourceMappingURL=index.js.map

package/dist/external/wikipage/wikimedia-client/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/external/wikipage/wikimedia-client/index.ts"],"sourcesContent":["export * from \"./core.js\";\nexport * from \"./html.js\";\nexport * from \"./wiki.js\";\n"],"mappings":"AAAA,cAAc;AACd,cAAc;AACd,cAAc;","names":[]}

package/dist/external/wikipage/wikimedia-client/json.d.ts

@@ -0,0 +1,7 @@
+declare function asRecord(value: unknown): Record<string, unknown>;
+declare function asArray(value: unknown): readonly unknown[];
+declare function getString(value: unknown): string | undefined;
+declare function getNumber(value: unknown): number | undefined;
+declare function getNestedString(value: Record<string, unknown>, path: readonly string[]): string | undefined;
+
+export { asArray, asRecord, getNestedString, getNumber, getString };

package/dist/external/wikipage/wikimedia-client/json.js

@@ -0,0 +1,27 @@
+function asRecord(value) {
+  return typeof value === "object" && value !== null ? value : {};
+}
+function asArray(value) {
+  return Array.isArray(value) ? value : [];
+}
+function getString(value) {
+  return typeof value === "string" && value !== "" ? value : void 0;
+}
+function getNumber(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : void 0;
+}
+function getNestedString(value, path) {
+  let current = value;
+  for (const part of path) {
+    current = asRecord(current)[part];
+  }
+  return getString(current);
+}
+export {
+  asArray,
+  asRecord,
+  getNestedString,
+  getNumber,
+  getString
+};
+//# sourceMappingURL=json.js.map

package/dist/external/wikipage/wikimedia-client/json.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/external/wikipage/wikimedia-client/json.ts"],"sourcesContent":["export function asRecord(value: unknown): Record<string, unknown> {\n  return typeof value === \"object\" && value !== null\n    ? (value as Record<string, unknown>)\n    : {};\n}\n\nexport function asArray(value: unknown): readonly unknown[] {\n  return Array.isArray(value) ? value : [];\n}\n\nexport function getString(value: unknown): string | undefined {\n  return typeof value === \"string\" && value !== \"\" ? value : undefined;\n}\n\nexport function getNumber(value: unknown): number | undefined {\n  return typeof value === \"number\" && Number.isFinite(value)\n    ? value\n    : undefined;\n}\n\nexport function getNestedString(\n  value: Record<string, unknown>,\n  path: readonly string[],\n): string | undefined {\n  let current: unknown = value;\n\n  for (const part of path) {\n    current = asRecord(current)[part];\n  }\n\n  return getString(current);\n}\n"],"mappings":"AAAO,SAAS,SAAS,OAAyC;AAChE,SAAO,OAAO,UAAU,YAAY,UAAU,OACzC,QACD,CAAC;AACP;AAEO,SAAS,QAAQ,OAAoC;AAC1D,SAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC;AACzC;AAEO,SAAS,UAAU,OAAoC;AAC5D,SAAO,OAAO,UAAU,YAAY,UAAU,KAAK,QAAQ;AAC7D;AAEO,SAAS,UAAU,OAAoC;AAC5D,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IACrD,QACA;AACN;AAEO,SAAS,gBACd,OACA,MACoB;AACpB,MAAI,UAAmB;AAEvB,aAAW,QAAQ,MAAM;AACvB,cAAU,SAAS,OAAO,EAAE,IAAI;AAAA,EAClC;AAEA,SAAO,UAAU,OAAO;AAC1B;","names":[]}