rotion 3.6.3 → 4.0.1

package/dist/exporter/dependencies.js

@@ -0,0 +1,102 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { notion, reqAPIWithBackoffAndCache } from './api.js';
+const collectors = new AsyncLocalStorage();
+/**
+ * recordDependency adds a dependency to the result that is being fetched.
+ */
+export function recordDependency(dependency) {
+    collectors.getStore()?.set(`${dependency.type}:${dependency.id}`, dependency);
+}
+/**
+ * recordDependencies adds the dependencies of a result that came from the
+ * cache, so that a result which contains it depends on them too.
+ */
+export function recordDependencies(dependencies) {
+    for (const dependency of dependencies ?? []) {
+        recordDependency(dependency);
+    }
+}
+/**
+ * collectDependencies runs fn and returns the dependencies recorded while it
+ * ran, including in nested calls. They are passed on to the caller's
+ * collector as well: the result of a nested FetchBlocks is copied into its
+ * parent's result, so the parent depends on the same pages.
+ */
+export async function collectDependencies(fn) {
+    const collector = new Map();
+    const value = await collectors.run(collector, fn);
+    const parent = collectors.getStore();
+    for (const [key, dependency] of collector) {
+        parent?.set(key, dependency);
+    }
+    return { value, dependencies: [...collector.values()] };
+}
+/**
+ * editedTimeOf returns the current last_edited_time of a page or database.
+ * The response is cached for ROTION_CACHE_AVAILABLE_DURATION, so checking
+ * the same dependency again and again costs one request in that period.
+ */
+export async function editedTimeOf(type, id) {
+    if (type === 'page') {
+        const page = await reqAPIWithBackoffAndCache({
+            name: 'notion.pages.retrieve',
+            func: notion().pages.retrieve,
+            args: { page_id: id },
+            count: 3,
+        });
+        return page.last_edited_time;
+    }
+    const db = await reqAPIWithBackoffAndCache({
+        name: 'notion.databases.retrieve',
+        func: notion().databases.retrieve,
+        args: { database_id: id },
+        count: 3,
+    });
+    return db.last_edited_time;
+}
+/**
+ * isAnyDependencyChanged tells whether a dependency has been edited since it
+ * was recorded. A dependency that can no longer be retrieved counts as
+ * changed, so that the result is fetched again and shows the failure.
+ */
+export async function isAnyDependencyChanged(dependencies) {
+    for (const dependency of dependencies ?? []) {
+        try {
+            if (await editedTimeOf(dependency.type, dependency.id) !== dependency.last_edited_time) {
+                return true;
+            }
+        }
+        catch {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * pageOfBlock returns the id of the page a block is in, following the
+ * parents of nested blocks. It returns undefined for a block that is not in
+ * a page, or one nested deeper than the limit.
+ */
+export async function pageOfBlock(block_id, limit = 20) {
+    let id = block_id;
+    for (let i = 0; i < limit; i++) {
+        const block = await reqAPIWithBackoffAndCache({
+            name: 'notion.blocks.retrieve',
+            func: notion().blocks.retrieve,
+            args: { block_id: id },
+            count: 3,
+        });
+        if (!('parent' in block)) {
+            return undefined;
+        }
+        if (block.parent.type === 'page_id') {
+            return block.parent.page_id;
+        }
+        if (block.parent.type !== 'block_id') {
+            return undefined;
+        }
+        id = block.parent.block_id;
+    }
+    return undefined;
+}
+//# sourceMappingURL=dependencies.js.map

package/dist/exporter/dependencies.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dependencies.js","sourceRoot":"","sources":["../../src/exporter/dependencies.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAA;AAQ5D,MAAM,UAAU,GAAG,IAAI,iBAAiB,EAA2B,CAAA;AAEnE;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAE,UAAsB;IACtD,UAAU,CAAC,QAAQ,EAAE,EAAE,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,EAAE,EAAE,EAAE,UAAU,CAAC,CAAA;AAC/E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAE,YAAsC;IACxE,KAAK,MAAM,UAAU,IAAI,YAAY,IAAI,EAAE,EAAE,CAAC;QAC5C,gBAAgB,CAAC,UAAU,CAAC,CAAA;IAC9B,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAK,EAAoB;IAChE,MAAM,SAAS,GAAG,IAAI,GAAG,EAAsB,CAAA;IAC/C,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;IACjD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAA;IACpC,KAAK,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1C,MAAM,EAAE,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAA;IAC9B,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,CAAA;AACzD,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAE,IAAwB,EAAE,EAAU;IACtE,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,yBAAyB,CAAoB;YAC9D,IAAI,EAAE,uBAAuB;YAC7B,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ;YAC7B,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;YACrB,KAAK,EAAE,CAAC;SACT,CAAC,CAAA;QACF,OAAO,IAAI,CAAC,gBAAgB,CAAA;IAC9B,CAAC;IACD,MAAM,EAAE,GAAG,MAAM,yBAAyB,CAAwB;QAChE,IAAI,EAAE,2BAA2B;QACjC,IAAI,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ;QACjC,IAAI,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE;QACzB,KAAK,EAAE,CAAC;KACT,CAAC,CAAA;IACF,OAAO,EAAE,CAAC,gBAAgB,CAAA;AAC5B,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAE,YAAsC;IAClF,KAAK,MAAM,UAAU,IAAI,YAAY,IAAI,EAAE,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,IAAI,MAAM,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC,KAAK,UAAU,CAAC,gBAAgB,EAAE,CAAC;gBACvF,OAAO,IAAI,CAAA;YACb,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAE,QAAgB,EAAE,KAAK,GAAG,EAAE;IAC7D,IAAI,EAAE,GAAG,QAAQ,CAAA;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAmB;YAC9D,IAAI,EAAE,wBAAwB;YAC9B,IAAI,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ;YAC9B,IAAI,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;YACtB,KAAK,EAAE,CAAC;SACT,CAAC,CAAA;QACF,IAAI,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,SAAS,CAAA;QAClB,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACpC,OAAO,KAAK,CAAC,MAAM,CAAC,OAAO,CAAA;QAC7B,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACrC,OAAO,SAAS,CAAA;QAClB,CAAC;QACD,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAA;IAC5B,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC"}

package/dist/exporter/failures.d.ts

@@ -0,0 +1,52 @@
+/**
+ * HTTPStatusError is thrown for a response whose status is not 2xx.
+ */
+export declare class HTTPStatusError extends Error {
+    status: number;
+    constructor(status: number, url: string);
+}
+/**
+ * Notion API errors that may succeed when the same request is sent again.
+ * They are retried, and a result that still misses something because of one
+ * of them is not cached.
+ */
+export declare const transientNotionErrorCodes: string[];
+/**
+ * isTransientError tells whether a failure may go away on the next attempt.
+ * A missing page, a restricted resource or a 404 stays the same however many
+ * times it is requested, so a result that lacks it can be cached. A rate
+ * limit, a server error, a timeout or a network error can not, and anything
+ * that is not known to be permanent counts as transient.
+ */
+export declare function isTransientError(error: unknown): boolean;
+/**
+ * StrictModeError is thrown by reportFailure when ROTION_STRICT is true.
+ */
+export declare class StrictModeError extends Error {
+    constructor(message: string, cause: unknown);
+}
+/**
+ * reportFailure is called for content that was caught failing and replaced
+ * with a fallback, such as an image without a local copy. It warns, and
+ * records the failure for collectFailures, which decides from the record
+ * whether its result is complete. With ROTION_STRICT=true it throws instead,
+ * so that a build does not publish a site with missing content.
+ */
+export declare function reportFailure(what: string, error: unknown): void;
+/**
+ * reportExtraFailure is reportFailure for extras that come from third
+ * parties: bookmark metadata, embeds and link previews. It only warns. One
+ * slow or rate-limited site should neither keep a page out of the cache nor
+ * fail a strict build.
+ */
+export declare function reportExtraFailure(what: string, error: unknown): void;
+/**
+ * collectFailures runs fn and tells whether a transient failure was reported
+ * while it ran, including in nested calls. Such a result is incomplete and
+ * must not be cached. The failures are passed on to the caller's collector,
+ * so that a result that contains this one is not cached either.
+ */
+export declare function collectFailures<T>(fn: () => Promise<T>): Promise<{
+    value: T;
+    complete: boolean;
+}>;

package/dist/exporter/failures.js

@@ -0,0 +1,105 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { isNotionClientError, APIErrorCode, ClientErrorCode, } from '@notionhq/client';
+import { warn } from './log.js';
+import { config } from './variables.js';
+/**
+ * HTTPStatusError is thrown for a response whose status is not 2xx.
+ */
+export class HTTPStatusError extends Error {
+    status;
+    constructor(status, url) {
+        super(`unexpected status ${status}: ${url}`);
+        this.name = 'HTTPStatusError';
+        this.status = status;
+    }
+}
+/**
+ * Notion API errors that may succeed when the same request is sent again.
+ * They are retried, and a result that still misses something because of one
+ * of them is not cached.
+ */
+export const transientNotionErrorCodes = [
+    APIErrorCode.RateLimited,
+    APIErrorCode.InternalServerError,
+    APIErrorCode.ServiceOverload,
+    APIErrorCode.ServiceUnavailable,
+    APIErrorCode.GatewayTimeout,
+    ClientErrorCode.ResponseError,
+    ClientErrorCode.RequestTimeout,
+];
+/**
+ * isTransientError tells whether a failure may go away on the next attempt.
+ * A missing page, a restricted resource or a 404 stays the same however many
+ * times it is requested, so a result that lacks it can be cached. A rate
+ * limit, a server error, a timeout or a network error can not, and anything
+ * that is not known to be permanent counts as transient.
+ */
+export function isTransientError(error) {
+    const seen = new Set();
+    let e = error;
+    while (e && typeof e === 'object' && !seen.has(e)) {
+        seen.add(e);
+        if (e instanceof HTTPStatusError) {
+            return e.status >= 500 || e.status === 408 || e.status === 429;
+        }
+        if (isNotionClientError(e)) {
+            return transientNotionErrorCodes.includes(e.code);
+        }
+        e = e.cause;
+    }
+    return true;
+}
+const collectors = new AsyncLocalStorage();
+/**
+ * StrictModeError is thrown by reportFailure when ROTION_STRICT is true.
+ */
+export class StrictModeError extends Error {
+    constructor(message, cause) {
+        super(message, { cause });
+        this.name = 'StrictModeError';
+    }
+}
+/**
+ * reportFailure is called for content that was caught failing and replaced
+ * with a fallback, such as an image without a local copy. It warns, and
+ * records the failure for collectFailures, which decides from the record
+ * whether its result is complete. With ROTION_STRICT=true it throws instead,
+ * so that a build does not publish a site with missing content.
+ */
+export function reportFailure(what, error) {
+    // Already reported where it happened, on its way out of an outer catch
+    if (error instanceof StrictModeError) {
+        throw error;
+    }
+    warn(`failed to get ${what}`, error);
+    if (config().strict) {
+        throw new StrictModeError(`failed to get ${what} (ROTION_STRICT=true)`, error);
+    }
+    if (isTransientError(error)) {
+        collectors.getStore()?.transient.push(error);
+    }
+}
+/**
+ * reportExtraFailure is reportFailure for extras that come from third
+ * parties: bookmark metadata, embeds and link previews. It only warns. One
+ * slow or rate-limited site should neither keep a page out of the cache nor
+ * fail a strict build.
+ */
+export function reportExtraFailure(what, error) {
+    warn(`failed to get ${what}`, error);
+}
+/**
+ * collectFailures runs fn and tells whether a transient failure was reported
+ * while it ran, including in nested calls. Such a result is incomplete and
+ * must not be cached. The failures are passed on to the caller's collector,
+ * so that a result that contains this one is not cached either.
+ */
+export async function collectFailures(fn) {
+    const collector = { transient: [] };
+    const value = await collectors.run(collector, fn);
+    if (collector.transient.length > 0) {
+        collectors.getStore()?.transient.push(...collector.transient);
+    }
+    return { value, complete: collector.transient.length === 0 };
+}
+//# sourceMappingURL=failures.js.map

package/dist/exporter/failures.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"failures.js","sourceRoot":"","sources":["../../src/exporter/failures.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EACL,mBAAmB,EACnB,YAAY,EACZ,eAAe,GAChB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAA;AAC/B,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAEvC;;GAEG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC,MAAM,CAAQ;IAEd,YAAa,MAAc,EAAE,GAAW;QACtC,KAAK,CAAC,qBAAqB,MAAM,KAAK,GAAG,EAAE,CAAC,CAAA;QAC5C,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAa;IACjD,YAAY,CAAC,WAAW;IACxB,YAAY,CAAC,mBAAmB;IAChC,YAAY,CAAC,eAAe;IAC5B,YAAY,CAAC,kBAAkB;IAC/B,YAAY,CAAC,cAAc;IAC3B,eAAe,CAAC,aAAa;IAC7B,eAAe,CAAC,cAAc;CAC/B,CAAA;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAE,KAAc;IAC9C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAW,CAAA;IAC/B,IAAI,CAAC,GAAY,KAAK,CAAA;IACtB,OAAO,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QACX,IAAI,CAAC,YAAY,eAAe,EAAE,CAAC;YACjC,OAAO,CAAC,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,CAAA;QAChE,CAAC;QACD,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,OAAO,yBAAyB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QACnD,CAAC;QACD,CAAC,GAAI,CAAyB,CAAC,KAAK,CAAA;IACtC,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAMD,MAAM,UAAU,GAAG,IAAI,iBAAiB,EAAa,CAAA;AAErD;;GAEG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC,YAAa,OAAe,EAAE,KAAc;QAC1C,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,CAAA;QACzB,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAA;IAC/B,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAE,IAAY,EAAE,KAAc;IACzD,uEAAuE;IACvE,IAAI,KAAK,YAAY,eAAe,EAAE,CAAC;QACrC,MAAM,KAAK,CAAA;IACb,CAAC;IACD,IAAI,CAAC,iBAAiB,IAAI,EAAE,EAAE,KAAK,CAAC,CAAA;IACpC,IAAI,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,IAAI,eAAe,CAAC,iBAAiB,IAAI,uBAAuB,EAAE,KAAK,CAAC,CAAA;IAChF,CAAC;IACD,IAAI,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,UAAU,CAAC,QAAQ,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC9C,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAE,IAAY,EAAE,KAAc;IAC9D,IAAI,CAAC,iBAAiB,IAAI,EAAE,EAAE,KAAK,CAAC,CAAA;AACtC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAK,EAAoB;IAC5D,MAAM,SAAS,GAAc,EAAE,SAAS,EAAE,EAAE,EAAE,CAAA;IAC9C,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;IACjD,IAAI,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,UAAU,CAAC,QAAQ,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,CAAA;IAC/D,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAA;AAC9D,CAAC"}

package/dist/exporter/files.d.ts

@@ -1,9 +1,30 @@
 import type { VideoBlockObjectResponseEx, EmbedBlockObjectResponseEx, HtmlMetadata, ImagePathWithSize } from './types.js';
 export declare const findLocationUrl: (rawHeaders: string[]) => string;
+/**
+ * getHTTP returns the body of a successful response. Any other status is an
+ * error, so that an error page is never taken for the requested content.
+ */
 export declare function getHTTP(reqUrl: string): Promise<string>;
 export declare function getJson<T>(reqUrl: string, httpFunc?: (reqUrl: string) => Promise<string>): Promise<T>;
 export declare const atoh: (a: string) => string;
 export declare const createDirWhenNotfound: (dir: string) => Promise<void>;
+/**
+ * markUsed sets the access time of files that Rotion has just used, so that
+ * pruneCache can tell them from files nothing uses any more. The modification
+ * time is kept, because it decides how long a cache file stays fresh. A file
+ * is marked at most once in markInterval by one process.
+ */
+export declare function markUsed(paths: string[]): Promise<void>;
+/**
+ * localFilesIn returns the paths of downloaded images and files that a cached
+ * result refers to, such as /images/block-...webp, relative to the docroot.
+ */
+export declare function localFilesIn(data: unknown, prefixes: string[]): string[];
+/**
+ * readCache returns a cached result. It marks the cache file and the images
+ * and files the result refers to as used: a page served from the cache still
+ * shows them, although nothing downloads them again.
+ */
 export declare function readCache<T>(f: string): Promise<T>;
 export declare function writeCache(f: string, data: unknown): Promise<void>;
 export declare function isAvailableCache(f: string, d?: number): Promise<boolean>;