rmapi-js 12.0.2 → 13.0.0

package/dist/raw.js

@@ -1,10 +1,24 @@
-import CRC32C from "crc-32/crc32c";
+// core-js installs these only when the runtime lacks them, so node 26+ keeps
+// its native implementations untouched
+import "core-js/modules/es.async-disposable-stack.constructor.js";
+import "core-js/modules/es.symbol.async-dispose.js";
+import "core-js/modules/es.uint8-array.from-base64.js";
+import "core-js/modules/es.uint8-array.from-hex.js";
+import "core-js/modules/es.uint8-array.to-base64.js";
+import "core-js/modules/es.uint8-array.to-hex.js";
+import CRC32C from "crc-32/crc32c.js";
 import { z } from "zod";
 import { ValidationError } from "./error.js";
-import { HEADER_LENGTH, parseV5, serializeRm, VERSION_PREFIX, } from "./rm5.js";
-import { parseRmScene } from "./rm6.js";
+import { HEADER_LENGTH, parseV5, serializeRm as serializeV5, VERSION_PREFIX, } from "./rm5.js";
+import { parseRmScene, serializeRmScene } from "./rm6.js";
 import { concatArrays } from "./utils.js";
 const hashReg = /^[0-9a-f]{64}$/;
+/** the dump format version; absent means the original text-only format */
+export const CACHE_VERSION = 2;
+/** marks a dumped cache entry stored as utf-8 text */
+export const TEXT_PREFIX = "t";
+/** marks a dumped cache entry stored as base64 */
+export const BYTES_PREFIX = "b";
 /**
  * parse the bytes of a reMarkable `.rm` page file
  *
@@ -34,6 +48,19 @@ export function parseRm(data) {
         throw new Error(`unsupported .lines version '${versionChar}'`);
     }
 }
+/**
+ * serialize a parsed reMarkable `.rm` page back to bytes
+ *
+ * The inverse of {@link parseRm | `parseRm`}: versions 3 and 5 render from a
+ * flat {@link RmPageV5 | `RmPageV5`}, version 6 from an
+ * {@link RmScene | `RmScene`}.
+ *
+ * @param page - the page to render
+ * @returns the `.rm` file bytes
+ */
+export function serializeRm(page) {
+    return page.version === 6 ? serializeRmScene(page) : serializeV5(page);
+}
 const tag = z
     .object({
     name: z.string(),
@@ -199,6 +226,33 @@ const content = z.union([
     documentContent,
     legacyDocumentContent,
 ]);
+const highlightsReg = /\.highlights\/[^/]+\.json$/;
+const highlightsFile = z
+    .object({
+    highlights: z.array(z.array(z
+        .object({
+        text: z.string(),
+        color: z.number().int(),
+        start: z.number().int(),
+        length: z.number().int(),
+        rects: z.array(z
+            .object({
+            x: z.number(),
+            y: z.number(),
+            width: z.number(),
+            height: z.number(),
+        })
+            .passthrough()),
+    })
+        .passthrough())),
+})
+    .passthrough();
+const pageMetadataReg = /\/[^/]+-metadata\.json$/;
+const pageMetadata = z
+    .object({
+    layers: z.array(z.object({ name: z.string() }).passthrough()),
+})
+    .passthrough();
 const metadata = z
     .object({
     lastModified: z.string().optional(),
@@ -306,9 +360,10 @@ function parseRawEntryLine(line) {
  * - `<docid>.content` - a json file roughly describing document properties (see {@link DocumentContent | `DocumentContent`})
  * - `<docid>.metadata` - metadata about the document (see {@link Metadata | `Metadata`})
  * - `<docid>.pagedata` - a text file where each line is the template of that page
+ * - `<docid>.template` - a template attached to the item (see {@link TemplateContent | `TemplateContent`})
  * - `<docid>/<pageid>.rm` - [speculative] raw remarkable vectors, text, etc
- * - `<docid>/<pageid>-metadata.json` - [speculative] metadata about the individual page
- * - `<docid>.highlights/<pageid>.json` - [speculative] highlights on the page
+ * - `<docid>/<pageid>-metadata.json` - page layer metadata (see {@link PageMetadata | `PageMetadata`})
+ * - `<docid>.highlights/<pageid>.json` - text highlights on the page (see {@link Highlight | `Highlight`})
  *
  * Some items will have both a `.pdf` and `.epub` file, likely due to preparing
  * for export. Collections only have `.content` and `.metadata` files, with
@@ -317,9 +372,12 @@ function parseRawEntryLine(line) {
  * ## Caching
  *
  * Since everything is tied to the hash of it's contents, we can agressively
- * cache results. We assume that text contents are "small" and so fully cache
- * them, where as binary files we treat as large and only store that we know
- * they exist to prevent future writes.
+ * cache results. Anything up to `maxCachedBytes` is kept as the bytes that were
+ * transferred; anything larger records only that the hash exists, which is
+ * enough to skip writing it again.
+ *
+ * A dump is tied to the account it came from, since a known hash is taken as
+ * proof the server already holds that file. Don't share one between accounts.
  *
  * By default, this only persists as long as the api instance is alive. However,
  * for performance reasons, you should call {@link dumpCache | `dumpCache`} to
@@ -336,20 +394,21 @@ export class RawRemarkable {
     /**
      * a cache of all hashes we know exist
      *
-     * The backend is a readonly file system of hashes to content. After a hash has
-     * been read or written successfully, we know it exists, and potentially it's
-     * contents. We don't want to cache large binary files, but we can cache the
-     * small text based metadata files. For binary files we write null, so we know
-     * not to write a a cached value again, but we'll still need to read it.
+     * The backend is a readonly file system of hashes to content, so once a hash
+     * has been read or written successfully we know it exists, and often what it
+     * holds. Contents are kept as the bytes that were transferred; anything over
+     * `maxCachedBytes` is stored as null, recording existence without the
+     * payload.
      */
     #cache;
-    constructor(authedFetch, cache, rawHost, uploadHost) {
+    #maxCachedBytes;
+    constructor(authedFetch, cache, rawHost, uploadHost, maxCachedBytes) {
         this.#authedFetch = authedFetch;
         this.#cache = cache;
         this.#rawHost = rawHost;
         this.#uploadHost = uploadHost;
+        this.#maxCachedBytes = maxCachedBytes;
     }
-    /** make an authorized request to remarkable */
     /**
      * gets the root hash and the current generation
      *
@@ -395,16 +454,12 @@ export class RawRemarkable {
     async getHash({ id: fileName, hash }) {
         const cached = this.#cache.get(hash);
         if (cached != null) {
-            const enc = new TextEncoder();
-            return enc.encode(cached);
+            return cached.slice();
         }
         else {
+            // NOTE two simultaneous requests will fetch twice
             const res = await this.#getHash(fileName, hash);
-            // mark that we know hash exists
-            const cacheVal = this.#cache.get(hash);
-            if (cacheVal === undefined) {
-                this.#cache.set(hash, null);
-            }
+            this.#cache.set(hash, res.byteLength <= this.#maxCachedBytes ? res.slice() : null);
             return res;
         }
     }
@@ -417,19 +472,8 @@ export class RawRemarkable {
      * @param ref - a reference to the stored file (see {@link getHash})
      * @returns the text
      */
-    async getText({ id: fileName, hash }) {
-        const cached = this.#cache.get(hash);
-        if (cached != null) {
-            return cached;
-        }
-        else {
-            // NOTE two simultaneous requests will fetch twice
-            const raw = await this.#getHash(fileName, hash);
-            const dec = new TextDecoder();
-            const res = dec.decode(raw);
-            this.#cache.set(hash, res);
-            return res;
-        }
+    async getText(ref) {
+        return new TextDecoder().decode(await this.getHash(ref));
     }
     /**
      * get the entries associated with a list hash
@@ -437,12 +481,12 @@ export class RawRemarkable {
      * A list hash is the root hash, or any hash with the type 80000000. NOTE
      * these are hashed differently than files.
      *
-     * @param ref - a reference whose `id` is `"root.docSchema"` for the root, or
-     *   `"<id>.docSchema"` for a sub-document's entry index
+     * @param ref - a reference whose `id` is the bare document id, or `"root"`
+     *   for the root index
      * @returns the entries
      */
-    async getEntries(ref) {
-        const rawFile = await this.getText(ref);
+    async getEntries({ id, hash }) {
+        const rawFile = await this.getText({ id: `${id}.docSchema`, hash });
         const [version, ...rest] = rawFile.slice(0, -1).split("\n");
         if (version === "3") {
             return { entries: rest.map(parseRawEntryLine) };
@@ -508,11 +552,44 @@ export class RawRemarkable {
         const bytes = await this.getHash(ref);
         return parseRm(bytes);
     }
+    /**
+     * get the parsed text highlights of a page
+     *
+     * @param ref - a reference to a `<docid>.highlights/<pageid>.json` file
+     * @returns the page's highlights; [speculative] each inner array groups the
+     *     fragments of one highlighted passage
+     */
+    async getHighlights(ref) {
+        const raw = await this.getText(ref);
+        const loaded = JSON.parse(raw);
+        return highlightsFile.parse(loaded).highlights;
+    }
+    /**
+     * get a template stored as a `<docid>.template` sidecar file
+     *
+     * @param ref - a reference to a `<docid>.template` file
+     * @returns the template content
+     */
+    async getTemplate(ref) {
+        const raw = await this.getText(ref);
+        const loaded = JSON.parse(raw);
+        return templateContent.parse(loaded);
+    }
+    /**
+     * get the parsed layer metadata of a page
+     *
+     * @param ref - a reference to a `<docid>/<pageid>-metadata.json` file
+     * @returns the page metadata
+     */
+    async getPageMetadata(ref) {
+        const raw = await this.getText(ref);
+        const loaded = JSON.parse(raw);
+        return pageMetadata.parse(loaded);
+    }
     /**
      * the same as {@link putFile | `putFile`} but rendering an `RmPage` to `.rm`
      * bytes
      *
-     * Only version 3 and 5 pages can be rendered; version 6 pages are read-only.
      */
     async putRm(fileName, page) {
         if (!fileName.endsWith(".rm")) {
@@ -578,67 +655,117 @@ export class RawRemarkable {
                     "x-goog-hash": `crc32c=${crcHash}`,
                 },
             });
-            // mark that we know this hash exists
-            const cacheVal = this.#cache.get(hash);
-            if (cacheVal === undefined) {
-                this.#cache.set(hash, null);
-            }
+            this.#cache.set(hash, bytes.byteLength <= this.#maxCachedBytes ? bytes.slice() : null);
         }
     }
     /**
-     * put a raw onto the server
+     * put a raw file onto the server
      *
-     * This returns the new expeced entry of the file you uploaded, and a promise
-     * to finish the upload successful. By splitting these two operations you can
-     * start using the uploaded entry while file finishes uploading.
+     * The returned entry is usable immediately, while the upload is still in
+     * flight; disposing it waits for the upload to finish. See
+     * {@link PendingEntry | `PendingEntry`}.
      *
      * NOTE: This won't update the state of the reMarkable until this entry is
      * incorporated into the root hash.
      *
      * @param fileName - the file name to upload (e.g. `<id>.pdf`)
      * @param bytes - the bytes to upload
-     * @returns the new entry and a promise to finish the upload
+     * @returns the new entry, pending its upload
      */
     async putFile(fileName, bytes) {
         const hash = await digest(bytes);
-        const res = {
+        const upload = this.#putFile(fileName, hash, bytes);
+        // an upload that fails before anyone disposes the entry would otherwise be
+        // an unhandled rejection; disposing re-awaits it and still throws
+        upload.catch(() => { });
+        return {
             id: fileName,
             hash,
             type: 0,
             subfiles: 0,
             size: bytes.length,
+            async [Symbol.asyncDispose]() {
+                await upload;
+            },
         };
-        return [res, this.#putFile(fileName, hash, bytes)];
     }
-    /** the same as {@link putFile | `putFile`} but with caching for text */
-    async putText(fileName, text) {
-        const enc = new TextEncoder();
-        const bytes = enc.encode(text);
-        const [ent, upload] = await this.putFile(fileName, bytes);
-        return [
-            ent,
-            upload.then(() => {
-                // on success, write to cache
-                this.#cache.set(ent.hash, text);
-            }),
-        ];
-    }
-    /** the same as {@link putText | `putText`} but with extra validation for Content */
+    async #putEncoded(fileName, text) {
+        return await this.putFile(fileName, new TextEncoder().encode(text));
+    }
+    /** the same as {@link putFile | `putFile`} but with extra validation for Content */
     async putContent(fileName, content) {
         if (!fileName.endsWith(".content")) {
             throw new Error(`fileName ${fileName} did not end with '.content'`);
         }
         else {
-            return await this.putText(fileName, JSON.stringify(content));
+            return await this.#putEncoded(fileName, JSON.stringify(content));
+        }
+    }
+    /**
+     * the same as {@link putFile | `putFile`} but for a `.pagedata` file
+     *
+     * @param fileName - the file to write, of the form `<docid>.pagedata`
+     * @param templates - one template name per page, in page order
+     */
+    async putPagedata(fileName, templates) {
+        if (!fileName.endsWith(".pagedata")) {
+            throw new Error(`fileName ${fileName} did not end with '.pagedata'`);
+        }
+        else {
+            return await this.#putEncoded(fileName, `${templates.join("\n")}\n`);
         }
     }
-    /** the same as {@link putText | `putText`} but with extra validation for Metadata */
+    /**
+     * the same as {@link putFile | `putFile`} but with extra validation for page
+     * layer metadata
+     *
+     * @param fileName - the file to write, of the form
+     *     `<docid>/<pageid>-metadata.json`
+     * @param meta - the page's layer metadata
+     */
+    async putPageMetadata(fileName, meta) {
+        if (!pageMetadataReg.test(fileName)) {
+            throw new ValidationError(fileName, pageMetadataReg, "fileName was not of the form '<docid>/<pageid>-metadata.json'");
+        }
+        else {
+            return await this.#putEncoded(fileName, JSON.stringify(meta));
+        }
+    }
+    /** the same as {@link putFile | `putFile`} but with extra validation for a template sidecar */
+    async putTemplate(fileName, template) {
+        if (!fileName.endsWith(".template")) {
+            throw new Error(`fileName ${fileName} did not end with '.template'`);
+        }
+        else {
+            return await this.#putEncoded(fileName, JSON.stringify(template));
+        }
+    }
+    /** the same as {@link putFile | `putFile`} but with extra validation for Metadata */
     async putMetadata(fileName, metadata) {
         if (!fileName.endsWith(".metadata")) {
             throw new Error(`fileName ${fileName} did not end with '.metadata'`);
         }
         else {
-            return await this.putText(fileName, JSON.stringify(metadata));
+            return await this.#putEncoded(fileName, JSON.stringify(metadata));
+        }
+    }
+    /**
+     * the same as {@link putFile | `putFile`} but with extra validation for
+     * highlights
+     *
+     * Rewraps the array in the file's `highlights` envelope, which
+     * {@link getHighlights | `getHighlights`} strips.
+     *
+     * @param fileName - the file to write, of the form
+     *     `<docid>.highlights/<pageid>.json`
+     * @param highlights - the page's highlights
+     */
+    async putHighlights(fileName, highlights) {
+        if (!highlightsReg.test(fileName)) {
+            throw new ValidationError(fileName, highlightsReg, "fileName was not of the form '<docid>.highlights/<pageid>.json'");
+        }
+        else {
+            return await this.#putEncoded(fileName, JSON.stringify({ highlights }));
         }
     }
     /**
@@ -658,11 +785,9 @@ export class RawRemarkable {
      * @param id - the id of the list to upload - this should be the item id if
      *   uploading an item list, or "root" if uploading a new root list. Note the
      *   asymmetry with {@link getEntries | `getEntries`}: `getEntries` takes the
-     *   full `"<id>.docSchema"` file name, whereas `putEntries` takes the bare id
-     *   and appends `.docSchema` (and special-cases `"root"`) itself.
      * @param entries - the entries to upload
      *
-     * @returns the new list entry and a promise to finish the upload
+     * @returns the new list entry, pending its upload
      */
     async putEntries(id, entries, schemaVersion) {
         if (id === "root" && schemaVersion === 3) {
@@ -699,14 +824,18 @@ export class RawRemarkable {
         else {
             throw new Error(`unsupported schema version ${schemaVersion}`);
         }
-        const res = {
+        const upload = this.#putFile(`${id}.docSchema`, hash, entryBuff);
+        upload.catch(() => { });
+        return {
             id,
             hash,
             type: schemaVersion > 3 ? 0 : 80000000,
             subfiles: sorted.length,
             size,
+            async [Symbol.asyncDispose]() {
+                await upload;
+            },
         };
-        return [res, this.#putFile(`${id}.docSchema`, hash, entryBuff)];
     }
     /**
      * upload a file to the reMarkable cloud using the simple api
@@ -744,7 +873,23 @@ export class RawRemarkable {
      * @returns a serialized version of the cache to pass to a new api instance
      */
     dumpCache() {
-        return JSON.stringify(Object.fromEntries(this.#cache));
+        const entries = {};
+        // utf-8 text is stored as itself; base64 would cost a third more for what
+        // is mostly json
+        const dec = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+        for (const [hash, value] of this.#cache) {
+            if (value === null) {
+                entries[hash] = null;
+                continue;
+            }
+            try {
+                entries[hash] = `${TEXT_PREFIX}${dec.decode(value)}`;
+            }
+            catch {
+                entries[hash] = `${BYTES_PREFIX}${value.toBase64()}`;
+            }
+        }
+        return JSON.stringify({ version: CACHE_VERSION, entries });
     }
     /** completely clear the cache */
     clearCache() {

package/dist/rm5.d.ts

@@ -62,7 +62,7 @@ export interface RmLayer {
  * a parsed version 3 or 5 page
  *
  * Coordinates use a top-left origin: `x` in `[0, width]`, `y` in `[0, height]`,
- * in device pixels. Only these pages can be re-rendered (via `raw.putRm`).
+ * in device pixels.
  */
 export interface RmPageV5 {
     /** the file format version */

package/dist/rm6.d.ts

@@ -5,8 +5,9 @@
  * of length-prefixed, tagged blocks. This module reads every block faithfully
  * — preserving `CrdtId`s, `LwwValue` wrappers, and each block's unread tail —
  * into an {@link RmScene | `RmScene`}, whose methods resolve the CRDT into
- * ordered layers, strokes, and text. Because nothing is dropped, the blocks
- * round-trip back to bytes.
+ * ordered layers, strokes, and text. Nothing is dropped, so
+ * {@link serializeRmScene | `serializeRmScene`} reproduces the original bytes
+ * exactly, including blocks it could not parse.
  *
  * @packageDocumentation
  */
@@ -44,7 +45,7 @@ export interface RmV6Point {
     y: number;
     /** pen speed */
     speed: number;
-    /** stroke width */
+    /** stroke width; integral for version 2 points, fractional for version 1 */
     width: number;
     /** pen direction, 0–255 mapping onto 0–2π */
     direction: number;
@@ -63,6 +64,10 @@ export interface RmV6Line {
     startingLength: number;
     /** the sampled points */
     points: RmV6Point[];
+    /** the stroke's timestamp id, if the file carried one */
+    timestampId?: CrdtId;
+    /** the stroke's move id, if the file carried one */
+    moveId?: CrdtId;
     /**
      * a packed little-endian uint32 color, only present for highlighter strokes
      *
@@ -128,6 +133,12 @@ export interface RmV6Text {
 }
 /** fields carried by every block */
 interface BlockCommon {
+    /**
+     * the header byte between the block length and the versions
+     *
+     * Zero in every well-formed file seen, but preserved rather than assumed.
+     */
+    reserved: number;
     /** the block's minimum reader version */
     minVersion: number;
     /** the block's current version (selects the point encoding for lines) */
@@ -167,14 +178,19 @@ export interface TreeNodeBlock extends BlockCommon {
     label: LwwValue<string>;
     /** whether the layer is visible */
     visible: LwwValue<boolean>;
-    /** the anchor node's crdt id, if anchored */
-    anchorId?: LwwValue<CrdtId>;
-    /** the anchor type, if anchored */
-    anchorType?: LwwValue<number>;
-    /** the anchor threshold, if anchored */
-    anchorThreshold?: LwwValue<number>;
-    /** the anchor's horizontal origin, if anchored */
-    anchorOriginX?: LwwValue<number>;
+    /** where the node is anchored, when it is */
+    anchor?: TreeNodeAnchor;
+}
+/** the four values a tree node carries when it's anchored, always together */
+export interface TreeNodeAnchor {
+    /** the anchor node's crdt id */
+    id: LwwValue<CrdtId>;
+    /** the anchor type */
+    type: LwwValue<number>;
+    /** the anchor threshold */
+    threshold: LwwValue<number>;
+    /** the anchor's horizontal origin */
+    originX: LwwValue<number>;
 }
 /** the `0x05` scene line item block — a stroke */
 export interface SceneLineItemBlock extends BlockCommon {
@@ -242,8 +258,8 @@ export interface PageInfoBlock extends BlockCommon {
     textCharsCount: number;
     /** the number of text lines on the page */
     textLinesCount: number;
-    /** the type-folio use count */
-    typeFolioUseCount: number;
+    /** the type-folio use count, if the block carried one */
+    typeFolioUseCount?: number;
 }
 /** the `0x09` author ids block — the author-id to uuid table */
 export interface AuthorIdsBlock extends BlockCommon {
@@ -273,6 +289,13 @@ export interface UnknownBlock extends BlockCommon {
     blockType: number;
     /** the raw block body bytes */
     data: Uint8Array;
+    /**
+     * the length the block header declared, when it overran the file
+     *
+     * Corrupt files exist in the wild; keeping the original figure lets them
+     * round-trip byte for byte instead of being silently rewritten.
+     */
+    declaredLength?: number;
 }
 /** any parsed version 6 block */
 export type RmBlock = MigrationInfoBlock | SceneTreeBlock | TreeNodeBlock | SceneLineItemBlock | SceneGlyphItemBlock | SceneGroupItemBlock | SceneTextItemBlock | SceneTombstoneItemBlock | RootTextBlock | PageInfoBlock | AuthorIdsBlock | SceneInfoBlock | UnknownBlock;
@@ -329,6 +352,8 @@ export declare class RmScene {
     /** the page's document text, if any */
     text(): RmV6Text | undefined;
 }
+/** serialize a parsed scene back to version 6 `.rm` bytes */
+export declare function serializeRmScene(scene: RmScene): Uint8Array;
 /** parse a version 6 `.rm` file into a resolvable scene */
 export declare function parseRmScene(data: Uint8Array): RmScene;
 export {};