rmapi-js 6.0.0 → 7.0.0

package/dist/index.js

@@ -4,7 +4,8 @@
  * After getting a device token with the {@link register | `register`} method,
  * persist it and create api instances using {@link remarkable | `remarkable`}.
  * Outside of registration, all relevant methods are in
- * {@link RemarkableApi | `RemarkableApi`}.
+ * {@link RemarkableApi | `RemarkableApi`}, or it's interior
+ * {@link RawRemarkableApi | `RawRemarkableApi`} (for lower level functions).
  *
  * @example
  * A simple rename
@@ -15,9 +16,10 @@
  * const token = await register(code)
  * // persist token
  * const api = await remarkable(token);
- * const [first, ...rest] = api.listfiles();
- * // rename first file
- * const api.rename(first.hash, "new name");
+ * // list all items (documents and collections)
+ * const [first, ...rest] = api.listItems();
+ * // rename first item
+ * const entry = api.rename(first.hash, "new name");
  * ```
  *
  * @example
@@ -36,29 +38,40 @@
  * which is a uuid4 and a hash, which indicates it's current state, and changes
  * as the item mutates, where the id is constant. Most mutable operations take
  * the initial hash so that merge conflicts can be resolved. Each entry has a
- * number of properties, but a key is the `parent`, which represents its parent
- * in the file structure. This will be another document id, or one of two
- * special ids, "" (the empty string) for the root directory, or "trash" for the
- * trash.
+ * number of properties, but a key property is the `parent`, which represents
+ * its parent in the file structure. This will be another document id, or one of
+ * two special ids, "" (the empty string) for the root directory, or "trash" for
+ * the trash.
+ *
+ * Detailed information about the low-level storage an apis can be found in
+ * {@link RawRemarkableApi | `RawRemarkableApi`}.
+ *
+ * Additionally, this entire api was reverse engineered, so some things are only
+ * `[speculative]`, or entirely `[unknown]`. If something breaks, please
+ * [file an issue!](https://github.com/erikbrinkman/rmapi-js/issues)
  *
  * @packageDocumentation
  */
 import { fromByteArray } from "base64-js";
-import { boolean, discriminator, elements, enumeration, float64, properties, string, values, } from "jtd-ts";
+import CRC32C from "crc-32/crc32c";
+import JSZip from "jszip";
+import { boolean, discriminator, elements, enumeration, float64, int32, nullable, properties, string, timestamp, uint32, uint8, values, } from "jtd-ts";
 import { v4 as uuid4 } from "uuid";
+import { LruCache } from "./lru";
 const AUTH_HOST = "https://webapp-prod.cloud.remarkable.engineering";
+const RAW_HOST = "https://eu.tectonic.remarkable.com";
 const SYNC_HOST = "https://web.eu.tectonic.remarkable.com";
 const idReg = /^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}||trash)$/;
 const hashReg = /^[0-9a-f]{64}$/;
-/** simple verification wrapper that allows for bypassing */
-function verification(res, schema, verify) {
-    if (!verify || schema.guard(res)) {
-        return res;
-    }
-    else {
-        throw new Error(`couldn't validate schema: ${JSON.stringify(res)} didn't match schema ${JSON.stringify(schema.schema())}`);
-    }
-}
+const tag = properties({
+    name: string(),
+    timestamp: float64(),
+}, undefined, true);
+const pageTag = properties({
+    name: string(),
+    pageId: string(),
+    timestamp: float64(),
+}, undefined, true);
 const commonProperties = {
     id: string(),
     hash: string(),
@@ -71,7 +84,7 @@ const commonOptionalProperties = {
     tags: elements(properties({
         name: string(),
         timestamp: float64(),
-    })),
+    }, undefined, true)),
 };
 const entry = discriminator("type", {
     CollectionType: properties(commonProperties, commonOptionalProperties, true),
@@ -85,11 +98,25 @@ const entries = elements(entry);
 const uploadEntry = properties({
     docID: string(),
     hash: string(),
-});
-const hashEntry = properties({ hash: string() });
+}, undefined, true);
+const hashEntry = properties({ hash: string() }, undefined, true);
 const hashesEntry = properties({
     hashes: values(string()),
-});
+}, undefined, true);
+/** An error that gets thrown when the backend while trying to update
+ *
+ * IF you encounter this error, you likely just need to try th request again. If
+ * you're trying to do several high-level `put` operations simultaneously,
+ * you'll likely encounter this error. You should either try to do them
+ * serially, or call the low level api directly to do one generation update.
+ *
+ * @see {@link RawRemarkableApi | `RawRemarkableApi`}
+ */
+export class GenerationError extends Error {
+    constructor() {
+        super("root generation was stale; try put again");
+    }
+}
 /** an error that results from a failed request */
 export class ResponseError extends Error {
     /** the response status number */
@@ -124,7 +151,7 @@ export class ValidationError extends Error {
  * @param code - the eight letter code a user got from `https://my.remarkable.com/device/browser/connect`.
  * @returns the device token necessary for creating an api instace. These never expire so persist as long as necessary.
  */
-export async function register(code, { deviceDesc = "browser-chrome", uuid = uuid4(), authHost = AUTH_HOST, fetch = globalThis.fetch, } = {}) {
+export async function register(code, { deviceDesc = "browser-chrome", uuid = uuid4(), authHost = AUTH_HOST, } = {}) {
     if (code.length !== 8) {
         throw new Error(`code should be length 8, but was ${code.length}`);
     }
@@ -146,19 +173,433 @@ export async function register(code, { deviceDesc = "browser-chrome", uuid = uui
         return await resp.text();
     }
 }
+const documentMetadata = properties(undefined, {
+    authors: elements(string()),
+    title: string(),
+    publicationDate: string(),
+    publisher: string(),
+}, true);
+const cPagePage = properties({
+    id: string(),
+    idx: properties({
+        timestamp: string(),
+        value: string(),
+    }, undefined, true),
+}, {
+    template: properties({
+        timestamp: string(),
+        value: string(),
+    }, undefined, true),
+    redir: properties({
+        timestamp: string(),
+        value: int32(),
+    }, undefined, true),
+    scrollTime: properties({
+        timestamp: string(),
+        value: timestamp(),
+    }, undefined, true),
+    verticalScroll: properties({
+        timestamp: string(),
+        value: float64(),
+    }, undefined, true),
+    deleted: properties({
+        timestamp: string(),
+        value: int32(),
+    }, undefined, true),
+}, true);
+const cPages = properties({
+    lastOpened: properties({
+        timestamp: string(),
+        value: string(),
+    }, undefined, true),
+    original: properties({
+        timestamp: string(),
+        value: int32(),
+    }, undefined, true),
+    pages: elements(cPagePage),
+    uuids: elements(properties({
+        first: string(),
+        second: uint32(),
+    }, undefined, true)),
+}, undefined, true);
+const collectionContent = properties(undefined, {
+    tags: elements(tag),
+});
+const documentContent = properties({
+    coverPageNumber: int32(),
+    documentMetadata,
+    extraMetadata: values(string()),
+    fileType: enumeration("epub", "notebook", "pdf"),
+    fontName: string(),
+    formatVersion: uint8(),
+    lineHeight: int32(),
+    margins: uint32(),
+    orientation: enumeration("portrait", "landscape"),
+    pageCount: uint32(),
+    sizeInBytes: string(),
+    textAlignment: enumeration("justify", "left"),
+    textScale: float64(),
+}, {
+    cPages,
+    customZoomCenterX: float64(),
+    customZoomCenterY: float64(),
+    customZoomOrientation: enumeration("portrait", "landscape"),
+    customZoomPageHeight: float64(),
+    customZoomPageWidth: float64(),
+    customZoomScale: float64(),
+    dummyDocument: boolean(),
+    keyboardMetadata: properties({
+        count: uint32(),
+        timestamp: float64(),
+    }, undefined, true),
+    lastOpenedPage: uint32(),
+    originalPageCount: int32(),
+    pages: elements(string()),
+    pageTags: elements(pageTag),
+    redirectionPageMap: elements(int32()),
+    tags: elements(tag),
+    transform: properties({
+        m11: float64(),
+        m12: float64(),
+        m13: float64(),
+        m21: float64(),
+        m22: float64(),
+        m23: float64(),
+        m31: float64(),
+        m32: float64(),
+        m33: float64(),
+    }, undefined, true),
+    // eslint-disable-next-line spellcheck/spell-checker
+    viewBackgroundFilter: enumeration("off", "fullpage"),
+    zoomMode: enumeration("bestFit", "customFit", "fitToHeight", "fitToWidth"),
+}, true);
+const metadata = properties({
+    lastModified: string(),
+    parent: string(),
+    pinned: boolean(),
+    type: enumeration("DocumentType", "CollectionType"),
+    visibleName: string(),
+}, {
+    lastOpened: string(),
+    lastOpenedPage: uint32(),
+    createdTime: string(),
+    deleted: boolean(),
+    metadatamodified: boolean(),
+    modified: boolean(),
+    synced: boolean(),
+    version: uint8(),
+}, true);
+const updatedRootHash = properties({
+    hash: string(),
+    generation: float64(),
+}, undefined, true);
+const rootHash = properties({
+    hash: string(),
+    generation: float64(),
+    schemaVersion: uint8(),
+}, undefined, true);
+async function digest(buff) {
+    const digest = await crypto.subtle.digest("SHA-256", buff);
+    return [...new Uint8Array(digest)]
+        .map((x) => x.toString(16).padStart(2, "0"))
+        .join("");
+}
+class RawRemarkable {
+    #authedFetch;
+    #rawHost;
+    /**
+     * a cache of all hashes we know exist
+     *
+     * The backend is a readonly file system of hashes to content. After a hash has
+     * been read or written successfully, we know it exists, and potentially it's
+     * contents. We don't want to cache large binary files, but we can cache the
+     * small text based metadata files. For binary files we write null, so we know
+     * not to write a a cached value again, but we'll still need to read it.
+     */
+    #cache;
+    constructor(authedFetch, cache, rawHost) {
+        this.#authedFetch = authedFetch;
+        this.#cache = cache;
+        this.#rawHost = rawHost;
+    }
+    /** make an authorized request to remarkable */
+    async getRootHash() {
+        const res = await this.#authedFetch("GET", `${this.#rawHost}/sync/v4/root`);
+        const raw = await res.text();
+        const loaded = JSON.parse(raw);
+        if (!rootHash.guardAssert(loaded))
+            throw Error("invalid root hash");
+        const { hash, generation, schemaVersion } = loaded;
+        if (schemaVersion !== 3) {
+            throw new Error(`schema version ${schemaVersion} not supported`);
+        }
+        else if (!Number.isSafeInteger(generation)) {
+            throw new Error(`generation ${generation} was not a safe integer; please file a bug report`);
+        }
+        else {
+            return [hash, generation];
+        }
+    }
+    async #getHash(hash) {
+        if (!hashReg.test(hash)) {
+            throw new ValidationError(hash, hashReg, "hash was not a valid hash");
+        }
+        const resp = await this.#authedFetch("GET", `${this.#rawHost}/sync/v3/files/${hash}`);
+        // TODO switch to `.bytes()`.
+        const raw = await resp.arrayBuffer();
+        return new Uint8Array(raw);
+    }
+    async getHash(hash) {
+        const cached = this.#cache.get(hash);
+        if (cached != null) {
+            const enc = new TextEncoder();
+            return enc.encode(cached);
+        }
+        else {
+            const res = await this.#getHash(hash);
+            // mark that we know hash exists
+            const cacheVal = this.#cache.get(hash);
+            if (cacheVal === undefined) {
+                this.#cache.set(hash, null);
+            }
+            return res;
+        }
+    }
+    async getText(hash) {
+        const cached = this.#cache.get(hash);
+        if (cached != null) {
+            return cached;
+        }
+        else {
+            // NOTE two simultaneous requests will fetch twice
+            const raw = await this.#getHash(hash);
+            const dec = new TextDecoder();
+            const res = dec.decode(raw);
+            this.#cache.set(hash, res);
+            return res;
+        }
+    }
+    async getEntries(hash) {
+        const rawFile = await this.getText(hash);
+        const [version, ...rest] = rawFile.slice(0, -1).split("\n");
+        if (version != "3") {
+            throw new Error(`schema version ${version} not supported`);
+        }
+        else {
+            return rest.map((line) => {
+                const [hash, type, id, subfiles, size] = line.split(":");
+                if (hash === undefined ||
+                    type === undefined ||
+                    id === undefined ||
+                    subfiles === undefined ||
+                    size === undefined) {
+                    throw new Error(`line '${line}' was not formatted correctly`);
+                }
+                else if (type === "80000000") {
+                    return {
+                        hash,
+                        type: 80000000,
+                        id,
+                        subfiles: parseInt(subfiles),
+                        size: parseInt(size),
+                    };
+                }
+                else if (type === "0" && subfiles === "0") {
+                    return {
+                        hash,
+                        type: 0,
+                        id,
+                        subfiles: 0,
+                        size: parseInt(size),
+                    };
+                }
+                else {
+                    throw new Error(`line '${line}' was not formatted correctly`);
+                }
+            });
+        }
+    }
+    async getContent(hash) {
+        const raw = await this.getText(hash);
+        const loaded = JSON.parse(raw);
+        // jtd can't verify non-discriminated unions, in this case, we have fileType
+        // defined or not. As a result, we only do a normal guard for the presence
+        // of tags (e.g. empty content or only specify tags). Otherwise we'll throw
+        // the full error for the richer content.
+        if (collectionContent.guard(loaded)) {
+            return loaded;
+        }
+        else if (documentContent.guardAssert(loaded)) {
+            return loaded;
+        }
+        else {
+            throw Error("invalid content");
+        }
+    }
+    async getMetadata(hash) {
+        const raw = await this.getText(hash);
+        const loaded = JSON.parse(raw);
+        if (!metadata.guardAssert(loaded))
+            throw Error("invalid metadata");
+        return loaded;
+    }
+    async putRootHash(hash, generation, broadcast = true) {
+        if (!Number.isSafeInteger(generation)) {
+            throw new Error(`generation ${generation} was not a safe integer`);
+        }
+        else if (!hashReg.test(hash)) {
+            throw new ValidationError(hash, hashReg, "rootHash was not a valid hash");
+        }
+        const body = JSON.stringify({
+            hash,
+            generation,
+            broadcast,
+        });
+        const resp = await this.#authedFetch("PUT", `${this.#rawHost}/sync/v3/root`, { body });
+        const raw = await resp.text();
+        const loaded = JSON.parse(raw);
+        if (!updatedRootHash.guardAssert(loaded))
+            throw Error("invalid root hash");
+        const { hash: newHash, generation: newGen } = loaded;
+        if (Number.isSafeInteger(newGen)) {
+            return [newHash, newGen];
+        }
+        else {
+            throw new Error(`new generation ${newGen} was not a safe integer; please file a bug report`);
+        }
+    }
+    async #putFile(hash, fileName, bytes) {
+        // if the hash is already in the cache, writing is pointless
+        if (!this.#cache.has(hash)) {
+            const crc = CRC32C.buf(bytes, 0);
+            const buff = new ArrayBuffer(4);
+            new DataView(buff).setInt32(0, crc, false);
+            const crcHash = fromByteArray(new Uint8Array(buff));
+            await this.#authedFetch("PUT", `${this.#rawHost}/sync/v3/files/${hash}`, {
+                body: bytes,
+                headers: {
+                    "rm-filename": fileName,
+                    // eslint-disable-next-line spellcheck/spell-checker
+                    "x-goog-hash": `crc32c=${crcHash}`,
+                },
+            });
+            // mark that we know this hash exists
+            const cacheVal = this.#cache.get(hash);
+            if (cacheVal === undefined) {
+                this.#cache.set(hash, null);
+            }
+        }
+    }
+    async putFile(id, bytes) {
+        const hash = await digest(bytes);
+        const res = {
+            id,
+            hash,
+            type: 0,
+            subfiles: 0,
+            size: bytes.length,
+        };
+        return [res, this.#putFile(hash, id, bytes)];
+    }
+    async putText(id, text) {
+        const enc = new TextEncoder();
+        const bytes = enc.encode(text);
+        const [ent, upload] = await this.putFile(id, bytes);
+        return [
+            ent,
+            upload.then(() => {
+                // on success, write to cache
+                this.#cache.set(ent.hash, text);
+            }),
+        ];
+    }
+    async putContent(id, content) {
+        if (!id.endsWith(".content")) {
+            throw new Error(`id ${id} did not end with '.content'`);
+        }
+        else {
+            return await this.putText(id, JSON.stringify(content));
+        }
+    }
+    async putMetadata(id, metadata) {
+        if (!id.endsWith(".metadata")) {
+            throw new Error(`id ${id} did not end with '.metadata'`);
+        }
+        else {
+            return await this.putText(id, JSON.stringify(metadata));
+        }
+    }
+    async putEntries(id, entries) {
+        // NOTE collections have a special hash function, the hash of their
+        // contents, so this needs to be different
+        entries.sort((a, b) => a.id.localeCompare(b.id));
+        const hashBuff = new Uint8Array(entries.length * 32);
+        for (const [start, { hash }] of entries.entries()) {
+            for (const [i, byte] of (hash.match(/../g) ?? []).entries()) {
+                hashBuff[start * 32 + i] = parseInt(byte, 16);
+            }
+        }
+        const hash = await digest(hashBuff);
+        const size = entries.reduce((acc, ent) => acc + ent.size, 0);
+        const records = ["3\n"];
+        for (const { hash, type, id, subfiles, size } of entries) {
+            records.push(`${hash}:${type}:${id}:${subfiles}:${size}\n`);
+        }
+        const res = {
+            id,
+            hash,
+            type: 80000000,
+            subfiles: entries.length,
+            size,
+        };
+        const enc = new TextEncoder();
+        return [
+            res,
+            // NOTE when monitoring requests, this had the extension .docSchema appended, but I'm not entirely sure why
+            this.#putFile(hash, `${id}.docSchema`, enc.encode(records.join(""))),
+        ];
+    }
+    dumpCache() {
+        return JSON.stringify(Object.fromEntries(this.#cache));
+    }
+    clearCache() {
+        this.#cache.clear();
+    }
+}
 /** the implementation of that api */
 class Remarkable {
     #userToken;
-    #fetch;
     #syncHost;
-    constructor(userToken, fetch, syncHost) {
+    /** the same cache that underlies the raw api, allowing us to modify it */
+    #cache;
+    raw;
+    #lastHashGen;
+    constructor(userToken, syncHost, rawHost, cache) {
         this.#userToken = userToken;
-        this.#fetch = fetch;
         this.#syncHost = syncHost;
+        this.#cache = cache;
+        this.raw = new RawRemarkable((method, url, { body, headers } = {}) => this.#authedFetch(url, { method, body, headers }), cache, rawHost);
+    }
+    async #getRootHash(refresh = false) {
+        if (refresh || this.#lastHashGen === undefined) {
+            this.#lastHashGen = await this.raw.getRootHash();
+        }
+        return this.#lastHashGen;
+    }
+    async #putRootHash(hash, generation) {
+        try {
+            this.#lastHashGen = await this.raw.putRootHash(hash, generation);
+        }
+        catch (ex) {
+            // if we hit a generation error, invalidate our cached generation
+            if (ex instanceof GenerationError) {
+                this.#lastHashGen = undefined;
+            }
+            throw ex;
+        }
     }
-    /** make an authorized request to remarkable */
     async #authedFetch(url, { body, method = "POST", headers = {}, }) {
-        const resp = await this.#fetch(url, {
+        const resp = await fetch(url, {
             method,
             headers: {
                 Authorization: `Bearer ${this.#userToken}`,
@@ -168,7 +609,12 @@ class Remarkable {
         });
         if (!resp.ok) {
             const msg = await resp.text();
-            throw new ResponseError(resp.status, resp.statusText, `failed reMarkable request: ${msg}`);
+            if (msg === '{"message":"precondition failed"}\n') {
+                throw new GenerationError();
+            }
+            else {
+                throw new ResponseError(resp.status, resp.statusText, `failed reMarkable request: ${msg}`);
+            }
         }
         else {
             return resp;
@@ -200,14 +646,149 @@ class Remarkable {
         const raw = await resp.text();
         return JSON.parse(raw);
     }
-    /** list all files */
-    async listFiles({ verify = true } = {}) {
+    /** list all items */
+    async listItems() {
         const res = await this.#fileRequest();
-        return verification(res, entries, verify);
+        if (!entries.guardAssert(res))
+            throw Error("invalid entries");
+        return res;
+    }
+    async listIds(refresh = false) {
+        const [hash] = await this.#getRootHash(refresh);
+        const entries = await this.raw.getEntries(hash);
+        return entries.map(({ id, hash }) => ({ id, hash }));
+    }
+    async getContent(hash) {
+        const entries = await this.raw.getEntries(hash);
+        const [cont] = entries.filter((e) => e.id.endsWith(".content"));
+        if (cont === undefined) {
+            throw new Error(`couldn't find contents for hash ${hash}`);
+        }
+        else {
+            return await this.raw.getContent(cont.hash);
+        }
+    }
+    async getMetadata(hash) {
+        const entries = await this.raw.getEntries(hash);
+        const [meta] = entries.filter((e) => e.id.endsWith(".metadata"));
+        if (meta === undefined) {
+            throw new Error(`couldn't find metadata for hash ${hash}`);
+        }
+        else {
+            return await this.raw.getMetadata(meta.hash);
+        }
+    }
+    async getPdf(hash) {
+        const entries = await this.raw.getEntries(hash);
+        const [pdf] = entries.filter((e) => e.id.endsWith(".pdf"));
+        if (pdf === undefined) {
+            throw new Error(`couldn't find pdf for hash ${hash}`);
+        }
+        else {
+            return await this.raw.getHash(pdf.hash);
+        }
+    }
+    async getEpub(hash) {
+        const entries = await this.raw.getEntries(hash);
+        const [epub] = entries.filter((e) => e.id.endsWith(".epub"));
+        if (epub === undefined) {
+            throw new Error(`couldn't find epub for hash ${hash}`);
+        }
+        else {
+            return await this.raw.getHash(epub.hash);
+        }
+    }
+    async getDocument(hash) {
+        const entries = await this.raw.getEntries(hash);
+        const zip = new JSZip();
+        for (const entry of entries) {
+            // TODO if this is .metadata we might want to assert type === "DocumentType"
+            zip.file(entry.id, this.raw.getHash(entry.hash));
+        }
+        return zip.generateAsync({ type: "uint8array" });
+    }
+    async #putFile(visibleName, fileType, buffer, { refresh, parent = "", pinned = false, zoomMode = "bestFit", viewBackgroundFilter, textScale = 1, textAlignment = "justify", fontName = "", coverPageNumber = -1, authors, title, publicationDate, publisher, extraMetadata = {}, lineHeight = -1, margins = 125, orientation = "portrait", tags, }) {
+        if (parent && !idReg.test(parent)) {
+            throw new ValidationError(parent, idReg, "parent must be a valid document id");
+        }
+        const id = uuid4();
+        const now = new Date();
+        const enc = new TextEncoder();
+        // upload raw files, and get root hash
+        const [[contentEntry, uploadContent], [metadataEntry, uploadMetadata], [pagedataEntry, uploadPagedata], [fileEntry, uploadFile], [rootHash, generation],] = await Promise.all([
+            this.raw.putContent(`${id}.content`, {
+                coverPageNumber,
+                documentMetadata: { authors, title, publicationDate, publisher },
+                extraMetadata,
+                lineHeight,
+                margins,
+                orientation,
+                fileType,
+                formatVersion: 1,
+                tags: tags?.map((name) => ({ name, timestamp: +now })) ?? [],
+                fontName,
+                textAlignment,
+                textScale,
+                zoomMode,
+                viewBackgroundFilter,
+                // NOTE for some reason we need to "fake" the number of pages at 1, and
+                // create "valid" output for that
+                originalPageCount: 1,
+                pageCount: 1,
+                pageTags: [],
+                pages: [uuid4()],
+                redirectionPageMap: [0],
+                sizeInBytes: buffer.length.toFixed(),
+            }),
+            this.raw.putMetadata(`${id}.metadata`, {
+                parent,
+                pinned,
+                lastModified: (+now).toFixed(),
+                createdTime: (+now).toFixed(),
+                type: "DocumentType",
+                visibleName,
+                lastOpened: "0",
+                lastOpenedPage: 0,
+            }),
+            // eslint-disable-next-line spellcheck/spell-checker
+            this.raw.putFile(`${id}.pagedata`, enc.encode("\n")),
+            this.raw.putFile(`${id}.${fileType}`, buffer),
+            this.#getRootHash(refresh),
+        ]);
+        // now fetch root entries and upload this file entry
+        const [[collectionEntry, uploadCollection], rootEntries] = await Promise.all([
+            this.raw.putEntries(id, [
+                contentEntry,
+                metadataEntry,
+                pagedataEntry,
+                fileEntry,
+            ]),
+            this.raw.getEntries(rootHash),
+        ]);
+        // now upload a new root entry
+        rootEntries.push(collectionEntry);
+        const [rootEntry, uploadRoot] = await this.raw.putEntries("root", rootEntries);
+        // before updating the root hash, first upload everything
+        await Promise.all([
+            uploadContent,
+            uploadMetadata,
+            uploadPagedata,
+            uploadFile,
+            uploadCollection,
+            uploadRoot,
+        ]);
+        await this.#putRootHash(rootEntry.hash, generation);
+        return { id, hash: collectionEntry.hash };
+    }
+    async putPdf(visibleName, buffer, opts = {}) {
+        return await this.#putFile(visibleName, "pdf", buffer, opts);
+    }
+    async putEpub(visibleName, buffer, opts = {}) {
+        return await this.#putFile(visibleName, "epub", buffer, opts);
     }
     /** upload a file */
-    async #uploadFile(parent, visibleName, buffer, contentType, verify) {
-        if (verify && !idReg.test(parent)) {
+    async #uploadFile(parent, visibleName, buffer, contentType) {
+        if (!idReg.test(parent)) {
             throw new ValidationError(parent, idReg, "parent must be a valid document id");
         }
         const res = await this.#fileRequest({
@@ -216,22 +797,26 @@ class Remarkable {
             contentType,
             body: buffer,
         });
-        return verification(res, uploadEntry, verify);
+        this.#lastHashGen = undefined; // clear the hash gen since this will change it
+        if (!uploadEntry.guardAssert(res))
+            throw Error("invalid upload entry");
+        const { hash, docID: id } = res;
+        return { hash, id };
     }
     /** create a folder */
-    async createFolder(visibleName, { parent = "", verify = true } = {}) {
-        return await this.#uploadFile(parent, visibleName, new ArrayBuffer(0), "folder", verify);
+    async createFolder(visibleName, { parent = "" } = {}) {
+        return await this.#uploadFile(parent, visibleName, new Uint8Array(0), "folder");
     }
     /** upload an epub */
-    async uploadEpub(visibleName, buffer, { parent = "", verify = true } = {}) {
-        return await this.#uploadFile(parent, visibleName, buffer, "application/epub+zip", verify);
+    async uploadEpub(visibleName, buffer, { parent = "" } = {}) {
+        return await this.#uploadFile(parent, visibleName, buffer, "application/epub+zip");
     }
     /** upload a pdf */
-    async uploadPdf(visibleName, buffer, { parent = "", verify = true } = {}) {
-        return await this.#uploadFile(parent, visibleName, buffer, "application/pdf", verify);
+    async uploadPdf(visibleName, buffer, { parent = "" } = {}) {
+        return await this.#uploadFile(parent, visibleName, buffer, "application/pdf");
     }
-    async #modify(hash, properties, verify) {
-        if (verify && !hashReg.test(hash)) {
+    async #modify(hash, properties) {
+        if (!hashReg.test(hash)) {
             throw new ValidationError(hash, hashReg, "hash to modify was not a valid hash");
         }
         // this does not allow setting pinned, although I don't know why
@@ -240,30 +825,31 @@ class Remarkable {
             body: JSON.stringify(properties),
             method: "PATCH",
         });
-        return verification(res, hashEntry, verify);
+        this.#lastHashGen = undefined; // clear the hash gen since this will change it
+        if (!hashEntry.guardAssert(res))
+            throw Error("invalid hash entry");
+        return res;
     }
     /** move an entry */
-    async move(hash, parent, { verify = true } = {}) {
-        if (verify && !idReg.test(parent)) {
+    async move(hash, parent) {
+        if (!idReg.test(parent)) {
             throw new ValidationError(parent, idReg, "parent must be a valid document id");
         }
-        return await this.#modify(hash, { parent }, verify);
+        return await this.#modify(hash, { parent });
     }
     /** delete an entry */
-    async delete(hash, opts = {}) {
-        return await this.move(hash, "trash", opts);
+    async delete(hash) {
+        return await this.move(hash, "trash");
     }
     /** rename an entry */
-    async rename(hash, visibleName, { verify = true } = {}) {
-        return await this.#modify(hash, { file_name: visibleName }, verify);
+    async rename(hash, visibleName) {
+        return await this.#modify(hash, { file_name: visibleName });
     }
     /** bulk modify hashes */
-    async #bulkModify(hashes, properties, verify) {
-        if (verify) {
-            const invalidHashes = hashes.filter((hash) => !hashReg.test(hash));
-            if (invalidHashes.length) {
-                throw new ValidationError(hashes.join(", "), hashReg, "hashes to modify were not a valid hashes");
-            }
+    async #bulkModify(hashes, properties) {
+        const invalidHashes = hashes.filter((hash) => !hashReg.test(hash));
+        if (invalidHashes.length) {
+            throw new ValidationError(hashes.join(", "), hashReg, "hashes to modify were not a valid hashes");
         }
         // this does not allow setting pinned, although I don't know why
         const res = await this.#fileRequest({
@@ -273,20 +859,59 @@ class Remarkable {
             }),
             method: "PATCH",
         });
-        return verification(res, hashesEntry, verify);
+        this.#lastHashGen = undefined; // clear the hash gen since this will change it
+        if (!hashesEntry.guardAssert(res))
+            throw Error("invalid hashes entry");
+        return res;
     }
     /** move many hashes */
-    async bulkMove(hashes, parent, { verify = true } = {}) {
-        if (verify && !idReg.test(parent)) {
+    async bulkMove(hashes, parent) {
+        if (!idReg.test(parent)) {
             throw new ValidationError(parent, idReg, "parent must be a valid document id");
         }
-        return await this.#bulkModify(hashes, { parent }, verify);
+        return await this.#bulkModify(hashes, { parent });
     }
     /** delete many hashes */
-    async bulkDelete(hashes, opts = {}) {
-        return await this.bulkMove(hashes, "trash", opts);
+    async bulkDelete(hashes) {
+        return await this.bulkMove(hashes, "trash");
+    }
+    // TODO ostensibly we could implement a bulk rename but idk why
+    /** dump the raw cache */
+    dumpCache() {
+        return this.raw.dumpCache();
+    }
+    async pruneCache(refresh) {
+        const [rootHash] = await this.#getRootHash(refresh);
+        // the keys to delete, we'll drop every key we can currently reach
+        const toDelete = new Set(this.#cache.keys());
+        // bfs through entries (to semi-optimize promise waiting, although this
+        // should only go one step) to track all hashes encountered
+        // NOTE that we could increase the cache in this process, or it's possible
+        // for other calls to increase the cache with misc values.
+        let entries = [await this.raw.getEntries(rootHash)];
+        let nextEntries = [];
+        while (entries.length) {
+            for (const entryList of entries) {
+                for (const { hash, type } of entryList) {
+                    toDelete.add(hash);
+                    if (type === 80000000) {
+                        nextEntries.push(this.raw.getEntries(hash));
+                    }
+                }
+            }
+            entries = await Promise.all(nextEntries);
+            nextEntries = [];
+        }
+        for (const key of toDelete) {
+            this.#cache.delete(key);
+        }
+    }
+    // finally remove any values we had in the cache initially, but couldn't reach
+    clearCache() {
+        this.raw.clearCache();
     }
 }
+const cached = values(nullable(string()));
 /**
  * create an instance of the api
  *
@@ -297,7 +922,7 @@ class Remarkable {
  *    registered. Create one with {@link register}.
  * @returns an api instance
  */
-export async function remarkable(deviceToken, { fetch = globalThis.fetch, authHost = AUTH_HOST, syncHost = SYNC_HOST, } = {}) {
+export async function remarkable(deviceToken, { authHost = AUTH_HOST, syncHost = SYNC_HOST, rawHost = RAW_HOST, cache, maxCacheSize = Infinity, } = {}) {
     const resp = await fetch(`${authHost}/token/json/2/user/new`, {
         method: "POST",
         headers: {
@@ -308,5 +933,15 @@ export async function remarkable(deviceToken, { fetch = globalThis.fetch, authHo
         throw new Error(`couldn't fetch auth token: ${resp.statusText}`);
     }
     const userToken = await resp.text();
-    return new Remarkable(userToken, fetch, syncHost);
+    const initCache = JSON.parse(cache ?? "{}");
+    if (cached.guard(initCache)) {
+        const entries = Object.entries(initCache);
+        const cache = maxCacheSize === Infinity
+            ? new Map(entries)
+            : new LruCache(maxCacheSize, entries);
+        return new Remarkable(userToken, syncHost, rawHost, cache);
+    }
+    else {
+        throw new Error("cache was not a valid cache (json string mapping); your cache must be corrupted somehow. Either initialize remarkable without a cache, or fix its format.");
+    }
 }