rotion 3.6.3 → 4.0.0

package/dist/exporter/github.js

@@ -1,15 +1,32 @@
 import { fetchWithTimeout } from './api.js';
 import { saveImage } from './files.js';
-import { debug } from './variables.js';
+import { config } from './variables.js';
+import { HTTPStatusError } from './failures.js';
 const reqInit = {
     headers: {
         'Accept': 'application/vnd.github+json',
         'X-GitHub-Api-Version': '2022-11-28',
     }
 };
+/**
+ * fetchGitHub returns the JSON body of a successful response from the GitHub
+ * API. Any other status throws an HTTPStatusError, instead of handing an
+ * error message to a caller that expects a repository or an issue. A request
+ * over the rate limit (403 or 429 with no requests remaining) is reported as
+ * 429, because it succeeds once the limit resets.
+ */
+export async function fetchGitHub(url) {
+    const res = await fetchWithTimeout(url, reqInit);
+    if (!res.ok) {
+        await res.body?.cancel();
+        const limited = (res.status === 403 || res.status === 429) && res.headers.get('x-ratelimit-remaining') === '0';
+        throw new HTTPStatusError(limited ? 429 : res.status, url);
+    }
+    return await res.json();
+}
 export async function getRepo({ owner, repo }, func) {
     const url = `https://api.github.com/repos/${owner}/${repo}`;
-    return (func) ? func(url, reqInit) : (await fetchWithTimeout(url, reqInit)).json();
+    return (func) ? func(url, reqInit) : fetchGitHub(url);
 }
 export async function getRepoForLinkPreview({ owner, repo }, func) {
     const res = await getRepo({ owner, repo }, func);
@@ -20,7 +37,7 @@ export async function getRepoForLinkPreview({ owner, repo }, func) {
         avatar_src = ipws.path;
     }
     catch (e) {
-        if (debug) {
+        if (config().debug) {
             console.log(`Failed to save github avatar: ${e}`);
         }
     }
@@ -34,7 +51,7 @@ export async function getRepoForLinkPreview({ owner, repo }, func) {
 }
 export async function getIssue({ owner, repo, number }, func) {
     const url = `https://api.github.com/repos/${owner}/${repo}/issues/${number}`;
-    return func ? func(url, reqInit) : (await fetchWithTimeout(url, reqInit)).json();
+    return func ? func(url, reqInit) : fetchGitHub(url);
 }
 export async function getIssueForLinkPreview({ owner, repo, number }, func) {
     const res = await getIssue({ owner, repo, number }, func);
@@ -46,7 +63,7 @@ export async function getIssueForLinkPreview({ owner, repo, number }, func) {
         avatar_src = ipws.path;
     }
     catch (e) {
-        if (debug) {
+        if (config().debug) {
             console.log(`Failed to save github avatar: ${e}`);
         }
     }

package/dist/exporter/github.js.map

@@ -1 +1 @@
-{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/exporter/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAA;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACtC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAMtC,MAAM,OAAO,GAAG;IACd,OAAO,EAAE;QACP,QAAQ,EAAE,6BAA6B;QACvC,sBAAsB,EAAE,YAAY;KACrC;CACF,CAAA;AAqHD,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,EAAkB,EAAE,IAAoC;IACjG,MAAM,GAAG,GAAG,gCAAgC,KAAK,IAAI,IAAI,EAAE,CAAA;IAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;AACpF,CAAC;AAUD,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,EAAE,KAAK,EAAE,IAAI,EAAkB,EAAE,IAAoC;IAC/G,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,IAAI,CAAC,CAAA;IAChD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,GAAG,GAAG,CAAA;IAC9D,IAAI,UAAU,GAAG,EAAE,CAAA;IACnB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,qBAAqB,CAAC,CAAA;QAC/D,UAAU,GAAG,IAAI,CAAC,IAAI,CAAA;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,EAAE,CAAC,CAAA;QACnD,CAAC;IACH,CAAC;IACD,OAAO;QACL,IAAI;QACJ,KAAK;QACL,UAAU;QACV,UAAU;QACV,UAAU;KACX,CAAA;AACH,CAAC;AA2GD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAmB,EAAE,IAAqC;IAC5G,MAAM,GAAG,GAAG,gCAAgC,KAAK,IAAI,IAAI,WAAW,MAAM,EAAE,CAAA;IAC5E,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;AAClF,CAAC;AAcD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAmB,EAAE,IAAqC;IAC1H,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,CAAA;IACzD,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAA;IACpB,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,GAAG,CAAA;IACzE,IAAI,UAAU,GAAG,EAAE,CAAA;IACnB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,qBAAqB,CAAC,CAAA;QAC/D,UAAU,GAAG,IAAI,CAAC,IAAI,CAAA;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,EAAE,CAAC,CAAA;QACnD,CAAC;IACH,CAAC;IACD,OAAO;QACL,KAAK;QACL,KAAK;QACL,UAAU;QACV,UAAU;QACV,UAAU;QACV,SAAS;QACT,SAAS,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QAC/D,KAAK,EAAE,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK;QAC5E,MAAM,EAAE,CAAC;KACV,CAAA;AACH,CAAC"}
+{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/exporter/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAA;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AACvC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAM/C,MAAM,OAAO,GAAG;IACd,OAAO,EAAE;QACP,QAAQ,EAAE,6BAA6B;QACvC,sBAAsB,EAAE,YAAY;KACrC;CACF,CAAA;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAK,GAAW;IAC/C,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IAChD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAA;QACxB,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,KAAK,GAAG,CAAA;QAC9G,MAAM,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC5D,CAAC;IACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAO,CAAA;AAC9B,CAAC;AAqHD,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,EAAkB,EAAE,IAAoC;IACjG,MAAM,GAAG,GAAG,gCAAgC,KAAK,IAAI,IAAI,EAAE,CAAA;IAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,CAAqB,GAAG,CAAC,CAAA;AAC3E,CAAC;AAUD,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,EAAE,KAAK,EAAE,IAAI,EAAkB,EAAE,IAAoC;IAC/G,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,IAAI,CAAC,CAAA;IAChD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,GAAG,GAAG,CAAA;IAC9D,IAAI,UAAU,GAAG,EAAE,CAAA;IACnB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,qBAAqB,CAAC,CAAA;QAC/D,UAAU,GAAG,IAAI,CAAC,IAAI,CAAA;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,EAAE,CAAC,CAAA;QACnD,CAAC;IACH,CAAC;IACD,OAAO;QACL,IAAI;QACJ,KAAK;QACL,UAAU;QACV,UAAU;QACV,UAAU;KACX,CAAA;AACH,CAAC;AA2GD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAmB,EAAE,IAAqC;IAC5G,MAAM,GAAG,GAAG,gCAAgC,KAAK,IAAI,IAAI,WAAW,MAAM,EAAE,CAAA;IAC5E,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,CAAsB,GAAG,CAAC,CAAA;AAC1E,CAAC;AAcD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAmB,EAAE,IAAqC;IAC1H,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,CAAA;IACzD,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAA;IACpB,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,GAAG,CAAA;IACzE,IAAI,UAAU,GAAG,EAAE,CAAA;IACnB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,qBAAqB,CAAC,CAAA;QAC/D,UAAU,GAAG,IAAI,CAAC,IAAI,CAAA;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,EAAE,CAAC,CAAA;QACnD,CAAC;IACH,CAAC;IACD,OAAO;QACL,KAAK;QACL,KAAK;QACL,UAAU;QACV,UAAU;QACV,UAAU;QACV,SAAS;QACT,SAAS,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QAC/D,KAAK,EAAE,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK;QAC5E,MAAM,EAAE,CAAC;KACV,CAAA;AACH,CAAC"}

package/dist/exporter/index.d.ts

@@ -4,3 +4,7 @@ export * from './page.js';
 export * from './types.js';
 export * from './breadcrumbs.js';
 export * from './validation.js';
+export { configure } from './variables.js';
+export type { RotionConfig } from './variables.js';
+export { pruneCache } from './prune.js';
+export type { PruneCacheArgs, PruneCacheRes } from './prune.js';

package/dist/exporter/index.js

@@ -4,4 +4,6 @@ export * from './page.js';
 export * from './types.js';
 export * from './breadcrumbs.js';
 export * from './validation.js';
+export { configure } from './variables.js';
+export { pruneCache } from './prune.js';
 //# sourceMappingURL=index.js.map

package/dist/exporter/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/exporter/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAA;AAC3B,cAAc,eAAe,CAAA;AAC7B,cAAc,WAAW,CAAA;AACzB,cAAc,YAAY,CAAA;AAC1B,cAAc,kBAAkB,CAAA;AAChC,cAAc,iBAAiB,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/exporter/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAA;AAC3B,cAAc,eAAe,CAAA;AAC7B,cAAc,WAAW,CAAA;AACzB,cAAc,YAAY,CAAA;AAC1B,cAAc,kBAAkB,CAAA;AAChC,cAAc,iBAAiB,CAAA;AAC/B,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAE1C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA"}

package/dist/exporter/log.d.ts

@@ -0,0 +1,6 @@
+/**
+ * warn prints one line to stderr for something that makes the output
+ * incomplete or slower, whether or not ROTION_DEBUG is set. Only the first
+ * line of the error message is printed; ROTION_DEBUG adds the whole error.
+ */
+export declare function warn(message: string, error?: unknown): void;

package/dist/exporter/log.js

@@ -0,0 +1,15 @@
+import { config } from './variables.js';
+const messageOf = (error) => error instanceof Error ? error.message : String(error);
+/**
+ * warn prints one line to stderr for something that makes the output
+ * incomplete or slower, whether or not ROTION_DEBUG is set. Only the first
+ * line of the error message is printed; ROTION_DEBUG adds the whole error.
+ */
+export function warn(message, error) {
+    const reason = error === undefined ? '' : `: ${messageOf(error).split('\n')[0]}`;
+    console.warn(`[rotion] ${message}${reason}`);
+    if (config().debug && error !== undefined) {
+        console.warn(error);
+    }
+}
+//# sourceMappingURL=log.js.map

package/dist/exporter/log.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"log.js","sourceRoot":"","sources":["../../src/exporter/log.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAEvC,MAAM,SAAS,GAAG,CAAC,KAAc,EAAU,EAAE,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAEpG;;;;GAIG;AACH,MAAM,UAAU,IAAI,CAAE,OAAe,EAAE,KAAe;IACpD,MAAM,MAAM,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAChF,OAAO,CAAC,IAAI,CAAC,YAAY,OAAO,GAAG,MAAM,EAAE,CAAC,CAAA;IAC5C,IAAI,MAAM,EAAE,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACrB,CAAC;AACH,CAAC"}

package/dist/exporter/mutex.js

@@ -1,6 +1,8 @@
 import { promises as fs } from 'fs';
 import path from 'path';
-import { cacheDir, debug } from './variables.js';
+import os from 'os';
+import { config } from './variables.js';
+import { warn } from './log.js';
 const DEFAULT_OPTIONS = {
     timeout: 600000, // 10 minutes — must accommodate long operations like
     // FetchDatabase that wrap many sequential image downloads
@@ -16,7 +18,7 @@ const DEFAULT_OPTIONS = {
  */
 export async function withFileLock(key, operation, options = {}) {
     const opts = { ...DEFAULT_OPTIONS, ...options };
-    const lockDir = path.join(cacheDir, 'locks');
+    const lockDir = path.join(config().cacheDir, 'locks');
     const lockFile = path.join(lockDir, `${key}.lock`);
     await fs.mkdir(lockDir, { recursive: true });
     const startTime = Date.now();
@@ -24,17 +26,18 @@ export async function withFileLock(key, operation, options = {}) {
     while (Date.now() - startTime < opts.timeout) {
         try {
             // Cleanup stale lock files
-            await cleanupStaleLock(lockFile, opts.maxAge);
+            await cleanupStaleLock(lockFile, opts.maxAge, opts.operationTimeout);
             // Create lock file exclusively
             fd = await fs.open(lockFile, 'wx');
-            // Write process ID and timestamp
+            // Write process ID, host and timestamp
             const lockData = JSON.stringify({
                 pid: process.pid,
+                hostname: os.hostname(),
                 timestamp: Date.now(),
                 key
             });
             await fd.writeFile(lockData);
-            if (debug) {
+            if (config().debug) {
                 console.log(`Lock acquired: ${key} (pid: ${process.pid})`);
             }
             // Execute operation, bounded by operationTimeout so a hanging
@@ -62,18 +65,37 @@ export async function withFileLock(key, operation, options = {}) {
 /**
  * Cleanup stale lock files
  */
-async function cleanupStaleLock(lockFile, maxAge) {
+async function cleanupStaleLock(lockFile, maxAge, operationTimeout) {
     try {
         const stats = await fs.stat(lockFile);
         const age = Date.now() - stats.mtime.getTime();
         if (age > maxAge) {
-            const lockData = await fs.readFile(lockFile, 'utf-8');
-            const lock = JSON.parse(lockData);
-            // Check if process is alive
-            if (!isProcessAlive(lock.pid)) {
+            const { pid, hostname } = await readLockHolder(lockFile);
+            // A lock holder writes its pid right after creating the file, so a lock
+            // older than maxAge without a readable pid was left by a process that
+            // died in between. Nothing will ever remove it, and without this it
+            // would hold every waiter until the timeout.
+            let reason;
+            if (pid === undefined) {
+                reason = 'no readable pid';
+            }
+            else if (hostname !== undefined && hostname !== os.hostname()) {
+                // A pid means nothing on another host that shares the cache directory
+                // (a volume shared by containers, for example): it may belong to an
+                // unrelated process here, or to none. The holder releases the lock
+                // within operationTimeout, so only a lock older than that is left
+                // behind.
+                if (age > operationTimeout) {
+                    reason = `held by ${hostname} for longer than ${operationTimeout}ms`;
+                }
+            }
+            else if (!isProcessAlive(pid)) {
+                reason = `dead pid: ${pid}`;
+            }
+            if (reason !== undefined) {
                 await fs.unlink(lockFile);
-                if (debug) {
-                    console.log(`Cleaned up stale lock: ${lockFile} (dead pid: ${lock.pid})`);
+                if (config().debug) {
+                    console.log(`Cleaned up stale lock: ${lockFile} (${reason})`);
                 }
             }
         }
@@ -82,6 +104,24 @@ async function cleanupStaleLock(lockFile, maxAge) {
         // Ignore if file doesn't exist or can't be read
     }
 }
+/**
+ * Read the pid and host a lock file was written with. Either is undefined
+ * when the file is empty or does not hold it; a lock written before the host
+ * was recorded is taken as one from this host.
+ */
+async function readLockHolder(lockFile) {
+    const lockData = await fs.readFile(lockFile, 'utf-8');
+    try {
+        const { pid, hostname } = JSON.parse(lockData);
+        return {
+            pid: Number.isInteger(pid) ? pid : undefined,
+            hostname: typeof hostname === 'string' ? hostname : undefined,
+        };
+    }
+    catch {
+        return {};
+    }
+}
 /**
  * Check if process is alive
  */
@@ -126,14 +166,13 @@ async function releaseLock(fd, lockFile, key) {
     try {
         await fd.close();
         await fs.unlink(lockFile);
-        if (debug) {
+        if (config().debug) {
             console.log(`Lock released: ${key} (pid: ${process.pid})`);
         }
     }
     catch (error) {
-        if (debug) {
-            console.error(`Failed to release lock: ${key}`, error);
-        }
+        // A lock file left behind makes the next caller wait until it is stale
+        warn(`failed to release the lock ${lockFile}`, error);
     }
 }
 //# sourceMappingURL=mutex.js.map

package/dist/exporter/mutex.js.map

@@ -1 +1 @@
-{"version":3,"file":"mutex.js","sourceRoot":"","sources":["../../src/exporter/mutex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,IAAI,CAAA;AACnC,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAahD,MAAM,eAAe,GAA0B;IAC7C,OAAO,EAAE,MAAM,EAAE,qDAAqD;IACrD,0DAA0D;IAC1D,oCAAoC;IACrD,aAAa,EAAE,GAAG,EAAE,QAAQ;IAC5B,MAAM,EAAE,KAAK,EAAE,WAAW;IAC1B,gBAAgB,EAAE,MAAM,EAAE,kDAAkD;IAClD,kDAAkD;IAClD,+CAA+C;CAC1E,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,SAA2B,EAC3B,OAAO,GAAgB,EAAE;IAEzB,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAA;IAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,OAAO,CAAC,CAAA;IAElD,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,EAAE,GAAyB,IAAI,CAAA;IAEnC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7C,IAAI,CAAC;YACH,2BAA2B;YAC3B,MAAM,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;YAE7C,+BAA+B;YAC/B,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;YAElC,iCAAiC;YACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;gBAC9B,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrB,GAAG;aACJ,CAAC,CAAA;YACF,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;YAE5B,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,UAAU,OAAO,CAAC,GAAG,GAAG,CAAC,CAAA;YAC5D,CAAC;YAED,8DAA8D;YAC9D,gDAAgD;YAChD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,SAAS,EAAE,IAAI,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAA;gBACnF,OAAO,MAAM,CAAA;YACf,CAAC;oBAAS,CAAC;gBACT,eAAe;gBACf,MAAM,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAA;YACtC,CAAC;QAEH,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC5B,4CAA4C;gBAC5C,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAA;gBACrE,SAAQ;YACV,CAAC;YACD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,YAAY,IAAI,CAAC,OAAO,IAAI,CAAC,CAAA;AACjF,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,QAAgB,EAAE,MAAc;IAC9D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAA;QAE9C,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;YACjB,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;YACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YAEjC,4BAA4B;YAC5B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9B,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;gBACzB,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,eAAe,IAAI,CAAC,GAAG,GAAG,CAAC,CAAA;gBAC3E,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gDAAgD;IAClD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,IAAI,CAAC;QACH,yEAAyE;QACzE,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;QACpB,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,uBAAuB,CACpC,SAA2B,EAC3B,gBAAwB,EACxB,GAAW;IAEX,IAAI,KAAiC,CAAA;IACrC,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;QAC7D,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YACtB,MAAM,CAAC,IAAI,KAAK,CAAC,4BAA4B,GAAG,WAAW,gBAAgB,IAAI,CAAC,CAAC,CAAA;QACnF,CAAC,EAAE,gBAAgB,CAAC,CAAA;QACpB,0DAA0D;QAC1D,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,UAAU;YAAE,KAAK,CAAC,KAAK,EAAE,CAAA;IACtD,CAAC,CAAC,CAAA;IACF,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,cAAc,CAAC,CAAC,CAAA;IAC1D,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAA;IAChC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,EAAiB,EAAE,QAAgB,EAAE,GAAW;IACzE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,KAAK,EAAE,CAAA;QAChB,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QACzB,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,UAAU,OAAO,CAAC,GAAG,GAAG,CAAC,CAAA;QAC5D,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,2BAA2B,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"mutex.js","sourceRoot":"","sources":["../../src/exporter/mutex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,IAAI,CAAA;AACnC,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,MAAM,IAAI,CAAA;AACnB,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAA;AAa/B,MAAM,eAAe,GAA0B;IAC7C,OAAO,EAAE,MAAM,EAAE,qDAAqD;IACrD,0DAA0D;IAC1D,oCAAoC;IACrD,aAAa,EAAE,GAAG,EAAE,QAAQ;IAC5B,MAAM,EAAE,KAAK,EAAE,WAAW;IAC1B,gBAAgB,EAAE,MAAM,EAAE,kDAAkD;IAClD,kDAAkD;IAClD,+CAA+C;CAC1E,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,SAA2B,EAC3B,OAAO,GAAgB,EAAE;IAEzB,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAA;IAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,OAAO,CAAC,CAAA;IAElD,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,EAAE,GAAyB,IAAI,CAAA;IAEnC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7C,IAAI,CAAC;YACH,2BAA2B;YAC3B,MAAM,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAEpE,+BAA+B;YAC/B,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;YAElC,uCAAuC;YACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;gBAC9B,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE;gBACvB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrB,GAAG;aACJ,CAAC,CAAA;YACF,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;YAE5B,IAAI,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,UAAU,OAAO,CAAC,GAAG,GAAG,CAAC,CAAA;YAC5D,CAAC;YAED,8DAA8D;YAC9D,gDAAgD;YAChD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,SAAS,EAAE,IAAI,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAA;gBACnF,OAAO,MAAM,CAAA;YACf,CAAC;oBAAS,CAAC;gBACT,eAAe;gBACf,MAAM,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAA;YACtC,CAAC;QAEH,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC5B,4CAA4C;gBAC5C,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAA;gBACrE,SAAQ;YACV,CAAC;YACD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,YAAY,IAAI,CAAC,OAAO,IAAI,CAAC,CAAA;AACjF,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,QAAgB,EAAE,MAAc,EAAE,gBAAwB;IACxF,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAA;QAE9C,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;YACjB,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAA;YAExD,wEAAwE;YACxE,sEAAsE;YACtE,oEAAoE;YACpE,6CAA6C;YAC7C,IAAI,MAA0B,CAAA;YAC9B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtB,MAAM,GAAG,iBAAiB,CAAA;YAC5B,CAAC;iBAAM,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAChE,sEAAsE;gBACtE,oEAAoE;gBACpE,mEAAmE;gBACnE,kEAAkE;gBAClE,UAAU;gBACV,IAAI,GAAG,GAAG,gBAAgB,EAAE,CAAC;oBAC3B,MAAM,GAAG,WAAW,QAAQ,oBAAoB,gBAAgB,IAAI,CAAA;gBACtE,CAAC;YACH,CAAC;iBAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,GAAG,aAAa,GAAG,EAAE,CAAA;YAC7B,CAAC;YAED,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;gBACzB,IAAI,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,KAAK,MAAM,GAAG,CAAC,CAAA;gBAC/D,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gDAAgD;IAClD,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC5C,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IACrD,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QAC9C,OAAO;YACL,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;YAC5C,QAAQ,EAAE,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;SAC9D,CAAA;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAA;IACX,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,IAAI,CAAC;QACH,yEAAyE;QACzE,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;QACpB,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,uBAAuB,CACpC,SAA2B,EAC3B,gBAAwB,EACxB,GAAW;IAEX,IAAI,KAAiC,CAAA;IACrC,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;QAC7D,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YACtB,MAAM,CAAC,IAAI,KAAK,CAAC,4BAA4B,GAAG,WAAW,gBAAgB,IAAI,CAAC,CAAC,CAAA;QACnF,CAAC,EAAE,gBAAgB,CAAC,CAAA;QACpB,0DAA0D;QAC1D,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,UAAU;YAAE,KAAK,CAAC,KAAK,EAAE,CAAA;IACtD,CAAC,CAAC,CAAA;IACF,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,cAAc,CAAC,CAAC,CAAA;IAC1D,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAA;IAChC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,EAAiB,EAAE,QAAgB,EAAE,GAAW;IACzE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,KAAK,EAAE,CAAA;QAChB,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QACzB,IAAI,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,UAAU,OAAO,CAAC,GAAG,GAAG,CAAC,CAAA;QAC5D,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,uEAAuE;QACvE,IAAI,CAAC,8BAA8B,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAA;IACvD,CAAC;AACH,CAAC"}

package/dist/exporter/page.js

@@ -1,13 +1,20 @@
 import { reqAPIWithBackoff, reqAPIWithBackoffAndCache, notion, } from './api.js';
-import { cacheDir, incrementalCache, debug, } from './variables.js';
+import { config } from './variables.js';
 import { createDirWhenNotfound, saveImage, readCache, writeCache, isEmpty, } from './files.js';
 import { withFileLock } from './mutex.js';
+import { collectFailures, reportFailure } from './failures.js';
+import { warn } from './log.js';
+// Only these property types answer pages.properties.retrieve with a list,
+// which is what page.meta collects. Any other type answers with a single
+// item, which page.meta leaves out, so asking for it would be a wasted request.
+const listPropertyTypes = new Set(['title', 'rich_text', 'people', 'relation', 'rollup']);
 /**
  * FetchPage retrieves page properties and download images in from properties.
  * And create cache that includes filepath of downloaded images.
  * The last_edited_time of 2nd args is for ROTION_INCREMENTAL_CACHE.
  */
 export const FetchPage = async ({ page_id, last_edited_time }) => {
+    const { cacheDir, incrementalCache, debug } = config();
     await createDirWhenNotfound(cacheDir);
     const cacheFile = `${cacheDir}/notion.pages.retrieve-${page_id}`;
     const lockKey = `page-${page_id}`;
@@ -35,37 +42,50 @@ export const FetchPage = async ({ page_id, last_edited_time }) => {
         catch (_) {
             /* not fatal */
         }
-        const page = await reqAPIWithBackoff({
-            func: notion.pages.retrieve,
-            args: { page_id },
-            count: 3
-        });
-        if ('properties' in page) {
-            let list;
-            for (const [, v] of Object.entries(page.properties)) {
-                const property_id = v.id;
-                const res = await reqAPIWithBackoffAndCache({
-                    name: 'notion.pages.properties.retrieve',
-                    func: notion.pages.properties.retrieve,
-                    args: { page_id, property_id },
-                    count: 3,
-                });
-                if (res.object !== 'list') {
-                    continue;
-                }
-                if (list === undefined) {
-                    list = res;
-                }
-                else {
-                    list.results.push(...res.results);
+        // A result that misses something because of a transient failure is not
+        // cached; see FetchBlocks.
+        const { value, complete } = await collectFailures(async () => {
+            const page = await reqAPIWithBackoff({
+                func: notion().pages.retrieve,
+                args: { page_id },
+                count: 3
+            });
+            if ('properties' in page) {
+                let list;
+                for (const [, v] of Object.entries(page.properties)) {
+                    if (!listPropertyTypes.has(v.type)) {
+                        continue;
+                    }
+                    const property_id = v.id;
+                    const res = await reqAPIWithBackoffAndCache({
+                        name: 'notion.pages.properties.retrieve',
+                        func: notion().pages.properties.retrieve,
+                        args: { page_id, property_id },
+                        count: 3,
+                    });
+                    if (res.object !== 'list') {
+                        continue;
+                    }
+                    if (list === undefined) {
+                        list = res;
+                    }
+                    else {
+                        list.results.push(...res.results);
+                    }
                 }
+                page.meta = list;
             }
-            page.meta = list;
+            await savePageCover(page);
+            await savePageIcon(page);
+            return page;
+        });
+        if (complete) {
+            await writeCache(cacheFile, value);
+        }
+        else {
+            warn(`not caching page ${page_id} because of a transient failure; it is fetched again on the next call`);
         }
-        await savePageCover(page);
-        await savePageIcon(page);
-        await writeCache(cacheFile, page);
-        return page;
+        return value;
     });
 };
 export async function savePageCover(page) {
@@ -83,9 +103,7 @@ export async function savePageCover(page) {
         }
     }
     catch (e) {
-        if (debug) {
-            console.log(`Failed to save page cover: ${e}`);
-        }
+        reportFailure(`cover of page ${page.id}`, e);
     }
 }
 export function getNotionIconUrl(icon) {
@@ -111,9 +129,7 @@ export async function savePageIcon(page) {
         }
     }
     catch (e) {
-        if (debug) {
-            console.log(`Failed to save page icon: ${e}`);
-        }
+        reportFailure(`icon of page ${page.id}`, e);
     }
 }
 //# sourceMappingURL=page.js.map

package/dist/exporter/page.js.map

@@ -1 +1 @@
-{"version":3,"file":"page.js","sourceRoot":"","sources":["../../src/exporter/page.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,yBAAyB,EACzB,MAAM,GACP,MAAM,UAAU,CAAA;AACjB,OAAO,EACL,QAAQ,EACR,gBAAgB,EAChB,KAAK,GACN,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACL,qBAAqB,EACrB,SAAS,EACT,SAAS,EACT,UAAU,EACV,OAAO,GACR,MAAM,YAAY,CAAA;AAOnB,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAUzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAiB,EAAyB,EAAE;IACrG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAA;IACrC,MAAM,SAAS,GAAG,GAAG,QAAQ,0BAA0B,OAAO,EAAE,CAAA;IAChE,MAAM,OAAO,GAAG,QAAQ,OAAO,EAAE,CAAA;IAEjC,OAAO,YAAY,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;QACtC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAoB,SAAS,CAAC,CAAA;YAC1D,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,IAAI,gBAAgB,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;oBACvD,IAAI,KAAK,EAAE,CAAC;wBACV,OAAO,CAAC,GAAG,CAAC,6BAA6B,SAAS,6EAA6E,CAAC,CAAA;oBAClI,CAAC;oBACD,OAAO,IAAI,CAAA;gBACb,CAAC;gBACD,IAAI,CAAC,gBAAgB,IAAI,CAAC,kBAAkB,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,KAAK,gBAAgB,CAAC,EAAE,CAAC;oBACpG,IAAI,KAAK,EAAE,CAAC;wBACV,OAAO,CAAC,GAAG,CAAC,sDAAsD,SAAS,EAAE,CAAC,CAAA;oBAChF,CAAC;oBACD,OAAO,IAAI,CAAA;gBACb,CAAC;gBACD,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,CAAC,GAAG,CAAC,yEAAyE,SAAS,EAAE,CAAC,CAAA;gBACnG,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,eAAe;QACjB,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAoB;YACtD,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;YAC3B,IAAI,EAAE,EAAE,OAAO,EAAE;YACjB,KAAK,EAAE,CAAC;SACT,CAAC,CAAA;QAEF,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,IAAI,IAAwC,CAAA;YAC5C,KAAK,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpD,MAAM,WAAW,GAAG,CAAC,CAAC,EAAE,CAAA;gBACxB,MAAM,GAAG,GAAG,MAAM,yBAAyB,CAA0B;oBACnE,IAAI,EAAE,kCAAkC;oBACxC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ;oBACtC,IAAI,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE;oBAC9B,KAAK,EAAE,CAAC;iBACT,CAAC,CAAA;gBACF,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;oBAC1B,SAAQ;gBACV,CAAC;gBACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACvB,IAAI,GAAG,GAAG,CAAA;gBACZ,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAA;gBACnC,CAAC;YACH,CAAC;YACD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAClB,CAAC;QAED,MAAM,aAAa,CAAC,IAAI,CAAC,CAAA;QACzB,MAAM,YAAY,CAAC,IAAI,CAAC,CAAA;QACxB,MAAM,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;QAEjC,OAAO,IAAI,CAAA;IACb,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAA8C;IAChF,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QACpD,OAAM;IACR,CAAC;IACD,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,cAAc,IAAI,CAAC,EAAE,EAAE,CAAC,CAAA;YAC9E,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;QAC5B,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,IAAI,CAAC,EAAE,EAAE,CAAC,CAAA;YAC1E,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;QAC5B,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,EAAE,CAAC,CAAA;QAChD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAqC;IACpE,OAAO,+BAA+B,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;AAC7G,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAA8C;IAC/E,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QAClD,OAAM;IACR,CAAC;IACD,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,aAAa,IAAI,CAAC,EAAE,EAAE,CAAC,CAAA;YAC5E,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;QAC3B,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,IAAI,CAAC,EAAE,EAAE,CAAC,CAAA;YACxE,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;QAC3B,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC5C,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,aAAa,IAAI,CAAC,EAAE,EAAE,CAAC,CACxD;YAAC,IAAY,CAAC,IAAI,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,CAAA;QAC/E,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,EAAE,CAAC,CAAA;QAC/C,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"page.js","sourceRoot":"","sources":["../../src/exporter/page.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,yBAAyB,EACzB,MAAM,GACP,MAAM,UAAU,CAAA;AACjB,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AACvC,OAAO,EACL,qBAAqB,EACrB,SAAS,EACT,SAAS,EACT,UAAU,EACV,OAAO,GACR,MAAM,YAAY,CAAA;AAOnB,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AACzC,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAA;AAE/B,0EAA0E;AAC1E,yEAAyE;AACzE,gFAAgF;AAChF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAA;AAUzF;;;;GAIG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAiB,EAAyB,EAAE;IACrG,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,KAAK,EAAE,GAAG,MAAM,EAAE,CAAA;IACtD,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAA;IACrC,MAAM,SAAS,GAAG,GAAG,QAAQ,0BAA0B,OAAO,EAAE,CAAA;IAChE,MAAM,OAAO,GAAG,QAAQ,OAAO,EAAE,CAAA;IAEjC,OAAO,YAAY,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;QACtC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAoB,SAAS,CAAC,CAAA;YAC1D,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,IAAI,gBAAgB,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;oBACvD,IAAI,KAAK,EAAE,CAAC;wBACV,OAAO,CAAC,GAAG,CAAC,6BAA6B,SAAS,6EAA6E,CAAC,CAAA;oBAClI,CAAC;oBACD,OAAO,IAAI,CAAA;gBACb,CAAC;gBACD,IAAI,CAAC,gBAAgB,IAAI,CAAC,kBAAkB,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,KAAK,gBAAgB,CAAC,EAAE,CAAC;oBACpG,IAAI,KAAK,EAAE,CAAC;wBACV,OAAO,CAAC,GAAG,CAAC,sDAAsD,SAAS,EAAE,CAAC,CAAA;oBAChF,CAAC;oBACD,OAAO,IAAI,CAAA;gBACb,CAAC;gBACD,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,CAAC,GAAG,CAAC,yEAAyE,SAAS,EAAE,CAAC,CAAA;gBACnG,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,eAAe;QACjB,CAAC;QAED,uEAAuE;QACvE,2BAA2B;QAC3B,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,eAAe,CAAC,KAAK,IAAI,EAAE;YAC3D,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAoB;gBACtD,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ;gBAC7B,IAAI,EAAE,EAAE,OAAO,EAAE;gBACjB,KAAK,EAAE,CAAC;aACT,CAAC,CAAA;YAEF,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;gBACzB,IAAI,IAAwC,CAAA;gBAC5C,KAAK,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBACpD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;wBACnC,SAAQ;oBACV,CAAC;oBACD,MAAM,WAAW,GAAG,CAAC,CAAC,EAAE,CAAA;oBACxB,MAAM,GAAG,GAAG,MAAM,yBAAyB,CAA0B;wBACnE,IAAI,EAAE,kCAAkC;wBACxC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ;wBACxC,IAAI,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE;wBAC9B,KAAK,EAAE,CAAC;qBACT,CAAC,CAAA;oBACF,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;wBAC1B,SAAQ;oBACV,CAAC;oBACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;wBACvB,IAAI,GAAG,GAAG,CAAA;oBACZ,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAA;oBACnC,CAAC;gBACH,CAAC;gBACD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;YAClB,CAAC;YAED,MAAM,aAAa,CAAC,IAAI,CAAC,CAAA;YACzB,MAAM,YAAY,CAAC,IAAI,CAAC,CAAA;YACxB,OAAO,IAAI,CAAA;QACb,CAAC,CAAC,CAAA;QAEF,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,oBAAoB,OAAO,uEAAuE,CAAC,CAAA;QAC1G,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAA8C;IAChF,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QACpD,OAAM;IACR,CAAC;IACD,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,cAAc,IAAI,CAAC,EAAE,EAAE,CAAC,CAAA;YAC9E,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;QAC5B,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,IAAI,CAAC,EAAE,EAAE,CAAC,CAAA;YAC1E,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;QAC5B,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,aAAa,CAAC,iBAAiB,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;IAC9C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAqC;IACpE,OAAO,+BAA+B,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;AAC7G,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAA8C;IAC/E,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QAClD,OAAM;IACR,CAAC;IACD,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,aAAa,IAAI,CAAC,EAAE,EAAE,CAAC,CAAA;YAC5E,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;QAC3B,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,IAAI,CAAC,EAAE,EAAE,CAAC,CAAA;YACxE,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;QAC3B,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC5C,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,aAAa,IAAI,CAAC,EAAE,EAAE,CAAC,CACxD;YAAC,IAAY,CAAC,IAAI,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,CAAA;QAC/E,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,aAAa,CAAC,gBAAgB,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;IAC7C,CAAC;AACH,CAAC"}

package/dist/exporter/prune.d.ts

@@ -0,0 +1,27 @@
+export interface PruneCacheArgs {
+    /** Remove what has not been used since this time */
+    before: Date;
+    /** List what would be removed without removing it */
+    dryRun?: boolean;
+}
+export interface PruneCacheRes {
+    /** Paths of the files removed, or that would be removed with dryRun */
+    removed: string[];
+}
+/**
+ * pruneCache removes cache files, downloaded images and downloaded files that
+ * Rotion has not used since `before`. Rotion marks what it uses by setting
+ * the access time: a cache file when it is read or written, a download when
+ * it is saved or found, and the downloads a cached result refers to when that
+ * result is read.
+ *
+ * Only files named the way Rotion names them are removed: cache files that
+ * start with "notion.", and downloads whose name ends with a hash. Other
+ * files in the same directories, such as images of the site itself, are
+ * kept. A download and its conversions (the WebP of an image, the PNG of a
+ * HEIC) are removed together, and only when none of them was used.
+ *
+ * Run it after a full build with the time the build started, or on a server
+ * with a period longer than any page is kept without asking Rotion again.
+ */
+export declare function pruneCache({ before, dryRun }: PruneCacheArgs): Promise<PruneCacheRes>;

package/dist/exporter/prune.js

@@ -0,0 +1,95 @@
+import { readdir, stat, unlink } from 'node:fs/promises';
+import path from 'node:path';
+import { config } from './variables.js';
+// A downloaded file is named <prefix>-<sha1 of the name in its URL><ext>,
+// and its conversions (.webp, .png) share the name without the extension.
+const downloadedName = /^(.+-[0-9a-f]{40})(\.[^.]+)?$/i;
+// Files a write left behind when it failed: <path>.<pid>.<time>.<random>.tmp
+const temporaryName = /\.\d+\.\d+\.[0-9a-z]+\.tmp$/;
+async function listFiles(dir) {
+    let names;
+    try {
+        names = await readdir(dir);
+    }
+    catch {
+        return [];
+    }
+    const files = [];
+    for (const name of names) {
+        const file = path.join(dir, name);
+        try {
+            const s = await stat(file);
+            if (s.isFile()) {
+                files.push({ name, file, atime: s.atimeMs, mtime: s.mtimeMs });
+            }
+        }
+        catch {
+            // Removed in the meantime
+        }
+    }
+    return files;
+}
+/**
+ * pruneCache removes cache files, downloaded images and downloaded files that
+ * Rotion has not used since `before`. Rotion marks what it uses by setting
+ * the access time: a cache file when it is read or written, a download when
+ * it is saved or found, and the downloads a cached result refers to when that
+ * result is read.
+ *
+ * Only files named the way Rotion names them are removed: cache files that
+ * start with "notion.", and downloads whose name ends with a hash. Other
+ * files in the same directories, such as images of the site itself, are
+ * kept. A download and its conversions (the WebP of an image, the PNG of a
+ * HEIC) are removed together, and only when none of them was used.
+ *
+ * Run it after a full build with the time the build started, or on a server
+ * with a period longer than any page is kept without asking Rotion again.
+ */
+export async function pruneCache({ before, dryRun = false }) {
+    const { cacheDir, docRoot, imageDir, fileDir } = config();
+    const limit = before.getTime();
+    const removed = [];
+    for (const f of await listFiles(cacheDir)) {
+        const unused = f.name.startsWith('notion.') && !temporaryName.test(f.name) && f.atime < limit;
+        const leftover = temporaryName.test(f.name) && f.mtime < limit;
+        if (unused || leftover) {
+            removed.push(f.file);
+        }
+    }
+    for (const dir of [path.join(docRoot, imageDir), path.join(docRoot, fileDir)]) {
+        const groups = new Map();
+        for (const f of await listFiles(dir)) {
+            if (temporaryName.test(f.name)) {
+                if (f.mtime < limit) {
+                    removed.push(f.file);
+                }
+                continue;
+            }
+            const m = f.name.match(downloadedName);
+            if (m === null) {
+                continue;
+            }
+            const group = groups.get(m[1]) ?? { files: [], used: 0 };
+            group.files.push(f.file);
+            group.used = Math.max(group.used, f.atime);
+            groups.set(m[1], group);
+        }
+        for (const group of groups.values()) {
+            if (group.used < limit) {
+                removed.push(...group.files);
+            }
+        }
+    }
+    if (!dryRun) {
+        for (const file of removed) {
+            try {
+                await unlink(file);
+            }
+            catch {
+                // Removed in the meantime
+            }
+        }
+    }
+    return { removed };
+}
+//# sourceMappingURL=prune.js.map

package/dist/exporter/prune.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prune.js","sourceRoot":"","sources":["../../src/exporter/prune.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAcvC,0EAA0E;AAC1E,0EAA0E;AAC1E,MAAM,cAAc,GAAG,gCAAgC,CAAA;AACvD,6EAA6E;AAC7E,MAAM,aAAa,GAAG,6BAA6B,CAAA;AAEnD,KAAK,UAAU,SAAS,CAAE,GAAW;IACnC,IAAI,KAAe,CAAA;IACnB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAA;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAA;IACX,CAAC;IACD,MAAM,KAAK,GAAG,EAAE,CAAA;IAChB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QACjC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAA;YAC1B,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBACf,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAE,EAAE,MAAM,EAAE,MAAM,GAAG,KAAK,EAAkB;IAC1E,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,EAAE,CAAA;IACzD,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,EAAE,CAAA;IAC9B,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,MAAM,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,KAAK,CAAA;QAC7F,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,KAAK,CAAA;QAC9D,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QACtB,CAAC;IACH,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC;QAC9E,MAAM,MAAM,GAAG,IAAI,GAAG,EAA6C,CAAA;QACnE,KAAK,MAAM,CAAC,IAAI,MAAM,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YACrC,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/B,IAAI,CAAC,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC;oBACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;gBACtB,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;YACtC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gBACf,SAAQ;YACV,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;YACxD,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YACxB,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAA;YAC1C,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;QACzB,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YACpC,IAAI,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,CAAC;gBACvB,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;YAC9B,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;YACpB,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,CAAA;AACpB,CAAC"}

package/dist/exporter/types.d.ts

@@ -278,10 +278,23 @@ export type SyncedBlockBlockObjectResponseEx = SyncedBlockBlockObjectResponse &
     children?: ListBlockChildrenResponseEx;
 };
 export type BlockObjectResponse = ParagraphBlockObjectResponseEx | Heading1BlockObjectResponse | Heading2BlockObjectResponse | Heading3BlockObjectResponse | BulletedListItemBlockObjectResponseEx | NumberedListItemBlockObjectResponseEx | QuoteBlockObjectResponse | ToDoBlockObjectResponse | ToggleBlockObjectResponseEx | TemplateBlockObjectResponse | SyncedBlockBlockObjectResponseEx | ChildPageBlockObjectResponseEx | ChildDatabaseBlockObjectResponseEx | EquationBlockObjectResponse | CodeBlockObjectResponse | CalloutBlockObjectResponseEx | DividerBlockObjectResponse | BreadcrumbBlockObjectResponseEx | TableOfContentsBlockObjectResponse | ColumnListBlockObjectResponseEx | ColumnBlockObjectResponse | LinkToPageBlockObjectResponse | TableBlockObjectResponseEx | TableRowBlockObjectResponse | EmbedBlockObjectResponseEx | BookmarkBlockObjectResponseEx | ImageBlockObjectResponseEx | VideoBlockObjectResponseEx | PdfBlockObjectResponseEx | FileBlockObjectResponseEx | AudioBlockObjectResponse | LinkPreviewBlockObjectResponseEx | UnsupportedBlockObjectResponse;
+/**
+ * Dependency is a page or database whose content is copied into a result of
+ * FetchBlocks: the source of a synced block, a mentioned page or database, a
+ * child page or a child database. Editing it does not change the
+ * last_edited_time of the page that shows it, so the incremental cache checks
+ * the dependency itself before it reuses the result.
+ */
+export interface Dependency {
+    type: 'page' | 'database';
+    id: string;
+    last_edited_time: string;
+}
 export type ListBlockChildrenResponseEx = ListBlockChildrenResponse & {
     results: Array<BlockObjectResponse>;
     children?: ListBlockChildrenResponse;
     last_edited_time?: string;
+    dependencies?: Dependency[];
 };
 export type LinkPreviewBlockObjectResponseEx = LinkPreviewBlockObjectResponse & {
     link_preview: {