rmapi-js 6.0.0 → 8.0.0

package/dist/index.js

@@ -4,7 +4,8 @@
  * After getting a device token with the {@link register | `register`} method,
  * persist it and create api instances using {@link remarkable | `remarkable`}.
  * Outside of registration, all relevant methods are in
- * {@link RemarkableApi | `RemarkableApi`}.
+ * {@link RemarkableApi | `RemarkableApi`}, or it's interior
+ * {@link RawRemarkableApi | `RawRemarkableApi`} (for lower level functions).
  *
  * @example
  * A simple rename
@@ -15,9 +16,10 @@
  * const token = await register(code)
  * // persist token
  * const api = await remarkable(token);
- * const [first, ...rest] = api.listfiles();
- * // rename first file
- * const api.rename(first.hash, "new name");
+ * // list all items (documents and collections)
+ * const [first, ...rest] = api.listItems();
+ * // rename first item
+ * const entry = api.rename(first.hash, "new name");
  * ```
  *
  * @example
@@ -27,7 +29,6 @@
  *
  * const api = await remarkable(...);
  * const entry = await api.putEpub("document name", epubBuffer);
- * await api.create(entry);
  * ```
  *
  * @remarks
@@ -36,60 +37,53 @@
  * which is a uuid4 and a hash, which indicates it's current state, and changes
  * as the item mutates, where the id is constant. Most mutable operations take
  * the initial hash so that merge conflicts can be resolved. Each entry has a
- * number of properties, but a key is the `parent`, which represents its parent
- * in the file structure. This will be another document id, or one of two
- * special ids, "" (the empty string) for the root directory, or "trash" for the
- * trash.
+ * number of properties, but a key property is the `parent`, which represents
+ * its parent in the file structure. This will be another document id, or one of
+ * two special ids, "" (the empty string) for the root directory, or "trash" for
+ * the trash.
+ *
+ * Detailed information about the low-level storage an apis can be found in
+ * {@link RawRemarkableApi | `RawRemarkableApi`}.
+ *
+ * Additionally, this entire api was reverse engineered, so some things are only
+ * `[speculative]`, or entirely `[unknown]`. If something breaks, please
+ * [file an issue!](https://github.com/erikbrinkman/rmapi-js/issues)
  *
  * @packageDocumentation
  */
 import { fromByteArray } from "base64-js";
-import { boolean, discriminator, elements, enumeration, float64, properties, string, values, } from "jtd-ts";
+import CRC32C from "crc-32/crc32c";
+import JSZip from "jszip";
+import { boolean, elements, enumeration, float64, int32, nullable, properties, string, timestamp, uint32, uint8, values, } from "jtd-ts";
 import { v4 as uuid4 } from "uuid";
+import { LruCache } from "./lru";
 const AUTH_HOST = "https://webapp-prod.cloud.remarkable.engineering";
-const SYNC_HOST = "https://web.eu.tectonic.remarkable.com";
+const RAW_HOST = "https://eu.tectonic.remarkable.com";
 const idReg = /^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}||trash)$/;
 const hashReg = /^[0-9a-f]{64}$/;
-/** simple verification wrapper that allows for bypassing */
-function verification(res, schema, verify) {
-    if (!verify || schema.guard(res)) {
-        return res;
-    }
-    else {
-        throw new Error(`couldn't validate schema: ${JSON.stringify(res)} didn't match schema ${JSON.stringify(schema.schema())}`);
+const tag = properties({
+    name: string(),
+    timestamp: float64(),
+}, undefined, true);
+const pageTag = properties({
+    name: string(),
+    pageId: string(),
+    timestamp: float64(),
+}, undefined, true);
+/** An error that gets thrown when the backend while trying to update
+ *
+ * IF you encounter this error, you likely just need to try th request again. If
+ * you're trying to do several high-level `put` operations simultaneously,
+ * you'll likely encounter this error. You should either try to do them
+ * serially, or call the low level api directly to do one generation update.
+ *
+ * @see {@link RawRemarkableApi | `RawRemarkableApi`}
+ */
+export class GenerationError extends Error {
+    constructor() {
+        super("root generation was stale; try put again");
     }
 }
-const commonProperties = {
-    id: string(),
-    hash: string(),
-    visibleName: string(),
-    lastModified: string(),
-    pinned: boolean(),
-};
-const commonOptionalProperties = {
-    parent: string(),
-    tags: elements(properties({
-        name: string(),
-        timestamp: float64(),
-    })),
-};
-const entry = discriminator("type", {
-    CollectionType: properties(commonProperties, commonOptionalProperties, true),
-    DocumentType: properties({
-        ...commonProperties,
-        lastOpened: string(),
-        fileType: enumeration("epub", "pdf", "notebook"),
-    }, commonOptionalProperties, true),
-});
-const entries = elements(entry);
-const uploadEntry = properties({
-    docID: string(),
-    hash: string(),
-});
-const hashEntry = properties({ hash: string() });
-const hashesEntry = properties({
-    hashes: values(string()),
-});
 /** an error that results from a failed request */
 export class ResponseError extends Error {
     /** the response status number */
@@ -114,6 +108,15 @@ export class ValidationError extends Error {
         this.regex = regex;
     }
 }
+/** an error that results while supplying a hash not found in the entries of the root hash */
+export class HashNotFoundError extends Error {
+    /** the hash that couldn't be found */
+    hash;
+    constructor(hash) {
+        super(`'${hash}' not found in the root hash`);
+        this.hash = hash;
+    }
+}
 /**
  * register a device and get the token needed to access the api
  *
@@ -124,7 +127,7 @@ export class ValidationError extends Error {
  * @param code - the eight letter code a user got from `https://my.remarkable.com/device/browser/connect`.
  * @returns the device token necessary for creating an api instace. These never expire so persist as long as necessary.
  */
-export async function register(code, { deviceDesc = "browser-chrome", uuid = uuid4(), authHost = AUTH_HOST, fetch = globalThis.fetch, } = {}) {
+export async function register(code, { deviceDesc = "browser-chrome", uuid = uuid4(), authHost = AUTH_HOST, } = {}) {
     if (code.length !== 8) {
         throw new Error(`code should be length 8, but was ${code.length}`);
     }
@@ -146,19 +149,431 @@ export async function register(code, { deviceDesc = "browser-chrome", uuid = uui
         return await resp.text();
     }
 }
+const documentMetadata = properties(undefined, {
+    authors: elements(string()),
+    title: string(),
+    publicationDate: string(),
+    publisher: string(),
+}, true);
+const cPagePage = properties({
+    id: string(),
+    idx: properties({
+        timestamp: string(),
+        value: string(),
+    }, undefined, true),
+}, {
+    template: properties({
+        timestamp: string(),
+        value: string(),
+    }, undefined, true),
+    redir: properties({
+        timestamp: string(),
+        value: int32(),
+    }, undefined, true),
+    scrollTime: properties({
+        timestamp: string(),
+        value: timestamp(),
+    }, undefined, true),
+    verticalScroll: properties({
+        timestamp: string(),
+        value: float64(),
+    }, undefined, true),
+    deleted: properties({
+        timestamp: string(),
+        value: int32(),
+    }, undefined, true),
+}, true);
+const cPages = properties({
+    lastOpened: properties({
+        timestamp: string(),
+        value: string(),
+    }, undefined, true),
+    original: properties({
+        timestamp: string(),
+        value: int32(),
+    }, undefined, true),
+    pages: elements(cPagePage),
+    uuids: elements(properties({
+        first: string(),
+        second: uint32(),
+    }, undefined, true)),
+}, undefined, true);
+const collectionContent = properties(undefined, {
+    tags: elements(tag),
+});
+const documentContent = properties({
+    coverPageNumber: int32(),
+    documentMetadata,
+    extraMetadata: values(string()),
+    fileType: enumeration("epub", "notebook", "pdf"),
+    fontName: string(),
+    formatVersion: uint8(),
+    lineHeight: int32(),
+    margins: uint32(),
+    orientation: enumeration("portrait", "landscape"),
+    pageCount: uint32(),
+    sizeInBytes: string(),
+    textAlignment: enumeration("justify", "left"),
+    textScale: float64(),
+}, {
+    cPages,
+    customZoomCenterX: float64(),
+    customZoomCenterY: float64(),
+    customZoomOrientation: enumeration("portrait", "landscape"),
+    customZoomPageHeight: float64(),
+    customZoomPageWidth: float64(),
+    customZoomScale: float64(),
+    dummyDocument: boolean(),
+    keyboardMetadata: properties({
+        count: uint32(),
+        timestamp: float64(),
+    }, undefined, true),
+    lastOpenedPage: uint32(),
+    originalPageCount: int32(),
+    pages: elements(string()),
+    pageTags: elements(pageTag),
+    redirectionPageMap: elements(int32()),
+    tags: elements(tag),
+    transform: properties({
+        m11: float64(),
+        m12: float64(),
+        m13: float64(),
+        m21: float64(),
+        m22: float64(),
+        m23: float64(),
+        m31: float64(),
+        m32: float64(),
+        m33: float64(),
+    }, undefined, true),
+    // eslint-disable-next-line spellcheck/spell-checker
+    viewBackgroundFilter: enumeration("off", "fullpage"),
+    zoomMode: enumeration("bestFit", "customFit", "fitToHeight", "fitToWidth"),
+}, true);
+const metadata = properties({
+    lastModified: string(),
+    parent: string(),
+    pinned: boolean(),
+    type: enumeration("DocumentType", "CollectionType"),
+    visibleName: string(),
+}, {
+    lastOpened: string(),
+    lastOpenedPage: uint32(),
+    createdTime: string(),
+    deleted: boolean(),
+    metadatamodified: boolean(),
+    modified: boolean(),
+    synced: boolean(),
+    version: uint8(),
+}, true);
+const updatedRootHash = properties({
+    hash: string(),
+    generation: float64(),
+}, undefined, true);
+const rootHash = properties({
+    hash: string(),
+    generation: float64(),
+    schemaVersion: uint8(),
+}, undefined, true);
+async function digest(buff) {
+    const digest = await crypto.subtle.digest("SHA-256", buff);
+    return [...new Uint8Array(digest)]
+        .map((x) => x.toString(16).padStart(2, "0"))
+        .join("");
+}
+class RawRemarkable {
+    #authedFetch;
+    #rawHost;
+    /**
+     * a cache of all hashes we know exist
+     *
+     * The backend is a readonly file system of hashes to content. After a hash has
+     * been read or written successfully, we know it exists, and potentially it's
+     * contents. We don't want to cache large binary files, but we can cache the
+     * small text based metadata files. For binary files we write null, so we know
+     * not to write a a cached value again, but we'll still need to read it.
+     */
+    #cache;
+    constructor(authedFetch, cache, rawHost) {
+        this.#authedFetch = authedFetch;
+        this.#cache = cache;
+        this.#rawHost = rawHost;
+    }
+    /** make an authorized request to remarkable */
+    async getRootHash() {
+        const res = await this.#authedFetch("GET", `${this.#rawHost}/sync/v4/root`);
+        const raw = await res.text();
+        const loaded = JSON.parse(raw);
+        if (!rootHash.guardAssert(loaded))
+            throw Error("invalid root hash");
+        const { hash, generation, schemaVersion } = loaded;
+        if (schemaVersion !== 3) {
+            throw new Error(`schema version ${schemaVersion} not supported`);
+        }
+        else if (!Number.isSafeInteger(generation)) {
+            throw new Error(`generation ${generation} was not a safe integer; please file a bug report`);
+        }
+        else {
+            return [hash, generation];
+        }
+    }
+    async #getHash(hash) {
+        if (!hashReg.test(hash)) {
+            throw new ValidationError(hash, hashReg, "hash was not a valid hash");
+        }
+        const resp = await this.#authedFetch("GET", `${this.#rawHost}/sync/v3/files/${hash}`);
+        // TODO switch to `.bytes()`.
+        const raw = await resp.arrayBuffer();
+        return new Uint8Array(raw);
+    }
+    async getHash(hash) {
+        const cached = this.#cache.get(hash);
+        if (cached != null) {
+            const enc = new TextEncoder();
+            return enc.encode(cached);
+        }
+        else {
+            const res = await this.#getHash(hash);
+            // mark that we know hash exists
+            const cacheVal = this.#cache.get(hash);
+            if (cacheVal === undefined) {
+                this.#cache.set(hash, null);
+            }
+            return res;
+        }
+    }
+    async getText(hash) {
+        const cached = this.#cache.get(hash);
+        if (cached != null) {
+            return cached;
+        }
+        else {
+            // NOTE two simultaneous requests will fetch twice
+            const raw = await this.#getHash(hash);
+            const dec = new TextDecoder();
+            const res = dec.decode(raw);
+            this.#cache.set(hash, res);
+            return res;
+        }
+    }
+    async getEntries(hash) {
+        const rawFile = await this.getText(hash);
+        const [version, ...rest] = rawFile.slice(0, -1).split("\n");
+        if (version != "3") {
+            throw new Error(`schema version ${version} not supported`);
+        }
+        else {
+            return rest.map((line) => {
+                const [hash, type, id, subfiles, size] = line.split(":");
+                if (hash === undefined ||
+                    type === undefined ||
+                    id === undefined ||
+                    subfiles === undefined ||
+                    size === undefined) {
+                    throw new Error(`line '${line}' was not formatted correctly`);
+                }
+                else if (type === "80000000") {
+                    return {
+                        hash,
+                        type: 80000000,
+                        id,
+                        subfiles: parseInt(subfiles),
+                        size: parseInt(size),
+                    };
+                }
+                else if (type === "0" && subfiles === "0") {
+                    return {
+                        hash,
+                        type: 0,
+                        id,
+                        subfiles: 0,
+                        size: parseInt(size),
+                    };
+                }
+                else {
+                    throw new Error(`line '${line}' was not formatted correctly`);
+                }
+            });
+        }
+    }
+    async getContent(hash) {
+        const raw = await this.getText(hash);
+        const loaded = JSON.parse(raw);
+        // jtd can't verify non-discriminated unions, in this case, we have fileType
+        // defined or not. As a result, we only do a normal guard for the presence
+        // of tags (e.g. empty content or only specify tags). Otherwise we'll throw
+        // the full error for the richer content.
+        if (collectionContent.guard(loaded)) {
+            return loaded;
+        }
+        else if (documentContent.guardAssert(loaded)) {
+            return loaded;
+        }
+        else {
+            throw Error("invalid content");
+        }
+    }
+    async getMetadata(hash) {
+        const raw = await this.getText(hash);
+        const loaded = JSON.parse(raw);
+        if (!metadata.guardAssert(loaded))
+            throw Error("invalid metadata");
+        return loaded;
+    }
+    async putRootHash(hash, generation, broadcast = true) {
+        if (!Number.isSafeInteger(generation)) {
+            throw new Error(`generation ${generation} was not a safe integer`);
+        }
+        else if (!hashReg.test(hash)) {
+            throw new ValidationError(hash, hashReg, "rootHash was not a valid hash");
+        }
+        const body = JSON.stringify({
+            hash,
+            generation,
+            broadcast,
+        });
+        const resp = await this.#authedFetch("PUT", `${this.#rawHost}/sync/v3/root`, { body });
+        const raw = await resp.text();
+        const loaded = JSON.parse(raw);
+        if (!updatedRootHash.guardAssert(loaded))
+            throw Error("invalid root hash");
+        const { hash: newHash, generation: newGen } = loaded;
+        if (Number.isSafeInteger(newGen)) {
+            return [newHash, newGen];
+        }
+        else {
+            throw new Error(`new generation ${newGen} was not a safe integer; please file a bug report`);
+        }
+    }
+    async #putFile(hash, fileName, bytes) {
+        // if the hash is already in the cache, writing is pointless
+        if (!this.#cache.has(hash)) {
+            const crc = CRC32C.buf(bytes, 0);
+            const buff = new ArrayBuffer(4);
+            new DataView(buff).setInt32(0, crc, false);
+            const crcHash = fromByteArray(new Uint8Array(buff));
+            await this.#authedFetch("PUT", `${this.#rawHost}/sync/v3/files/${hash}`, {
+                body: bytes,
+                headers: {
+                    "rm-filename": fileName,
+                    // eslint-disable-next-line spellcheck/spell-checker
+                    "x-goog-hash": `crc32c=${crcHash}`,
+                },
+            });
+            // mark that we know this hash exists
+            const cacheVal = this.#cache.get(hash);
+            if (cacheVal === undefined) {
+                this.#cache.set(hash, null);
+            }
+        }
+    }
+    async putFile(id, bytes) {
+        const hash = await digest(bytes);
+        const res = {
+            id,
+            hash,
+            type: 0,
+            subfiles: 0,
+            size: bytes.length,
+        };
+        return [res, this.#putFile(hash, id, bytes)];
+    }
+    async putText(id, text) {
+        const enc = new TextEncoder();
+        const bytes = enc.encode(text);
+        const [ent, upload] = await this.putFile(id, bytes);
+        return [
+            ent,
+            upload.then(() => {
+                // on success, write to cache
+                this.#cache.set(ent.hash, text);
+            }),
+        ];
+    }
+    async putContent(id, content) {
+        if (!id.endsWith(".content")) {
+            throw new Error(`id ${id} did not end with '.content'`);
+        }
+        else {
+            return await this.putText(id, JSON.stringify(content));
+        }
+    }
+    async putMetadata(id, metadata) {
+        if (!id.endsWith(".metadata")) {
+            throw new Error(`id ${id} did not end with '.metadata'`);
+        }
+        else {
+            return await this.putText(id, JSON.stringify(metadata));
+        }
+    }
+    async putEntries(id, entries) {
+        // NOTE collections have a special hash function, the hash of their
+        // contents, so this needs to be different
+        entries.sort((a, b) => a.id.localeCompare(b.id));
+        const hashBuff = new Uint8Array(entries.length * 32);
+        for (const [start, { hash }] of entries.entries()) {
+            for (const [i, byte] of (hash.match(/../g) ?? []).entries()) {
+                hashBuff[start * 32 + i] = parseInt(byte, 16);
+            }
+        }
+        const hash = await digest(hashBuff);
+        const size = entries.reduce((acc, ent) => acc + ent.size, 0);
+        const records = ["3\n"];
+        for (const { hash, type, id, subfiles, size } of entries) {
+            records.push(`${hash}:${type}:${id}:${subfiles}:${size}\n`);
+        }
+        const res = {
+            id,
+            hash,
+            type: 80000000,
+            subfiles: entries.length,
+            size,
+        };
+        const enc = new TextEncoder();
+        return [
+            res,
+            // NOTE when monitoring requests, this had the extension .docSchema appended, but I'm not entirely sure why
+            this.#putFile(hash, `${id}.docSchema`, enc.encode(records.join(""))),
+        ];
+    }
+    dumpCache() {
+        return JSON.stringify(Object.fromEntries(this.#cache));
+    }
+    clearCache() {
+        this.#cache.clear();
+    }
+}
 /** the implementation of that api */
 class Remarkable {
     #userToken;
-    #fetch;
-    #syncHost;
-    constructor(userToken, fetch, syncHost) {
+    /** the same cache that underlies the raw api, allowing us to modify it */
+    #cache;
+    raw;
+    #lastHashGen;
+    constructor(userToken, rawHost, cache) {
         this.#userToken = userToken;
-        this.#fetch = fetch;
-        this.#syncHost = syncHost;
+        this.#cache = cache;
+        this.raw = new RawRemarkable((method, url, { body, headers } = {}) => this.#authedFetch(url, { method, body, headers }), cache, rawHost);
+    }
+    async #getRootHash(refresh = false) {
+        if (refresh || this.#lastHashGen === undefined) {
+            this.#lastHashGen = await this.raw.getRootHash();
+        }
+        return this.#lastHashGen;
+    }
+    async #putRootHash(hash, generation) {
+        try {
+            this.#lastHashGen = await this.raw.putRootHash(hash, generation);
+        }
+        catch (ex) {
+            // if we hit a generation error, invalidate our cached generation
+            if (ex instanceof GenerationError) {
+                this.#lastHashGen = undefined;
+            }
+            throw ex;
+        }
     }
-    /** make an authorized request to remarkable */
     async #authedFetch(url, { body, method = "POST", headers = {}, }) {
-        const resp = await this.#fetch(url, {
+        const resp = await fetch(url, {
             method,
             headers: {
                 Authorization: `Bearer ${this.#userToken}`,
@@ -168,125 +583,365 @@ class Remarkable {
         });
         if (!resp.ok) {
             const msg = await resp.text();
-            throw new ResponseError(resp.status, resp.statusText, `failed reMarkable request: ${msg}`);
+            if (msg === '{"message":"precondition failed"}\n') {
+                throw new GenerationError();
+            }
+            else {
+                throw new ResponseError(resp.status, resp.statusText, `failed reMarkable request: ${msg}`);
+            }
         }
         else {
             return resp;
         }
     }
-    /** a generic request to the new files api
-     *
-     * @param meta - remarkable metadata to set, often json formatted or empty
-     * @param method - the http method to use
-     * @param contentType - the http content type to set
-     * @param body - body content, often raw bytes or json
-     * @param hash - the hash of a specific file to target
-     */
-    async #fileRequest({ meta = "", method = "GET", 
-    // eslint-disable-next-line spellcheck/spell-checker
-    contentType = "text/plain;charset=UTF-8", body, hash, } = {}) {
-        const encoder = new TextEncoder();
-        const encMeta = encoder.encode(meta);
-        const suffix = hash === undefined ? "" : `/${hash}`;
-        const resp = await this.#authedFetch(`${this.#syncHost}/doc/v2/files${suffix}`, {
-            body,
-            method,
-            headers: {
-                "content-type": contentType,
-                "rm-meta": fromByteArray(encMeta),
-                "rm-source": "WebLibrary",
-            },
-        });
-        const raw = await resp.text();
-        return JSON.parse(raw);
+    async #convertEntry({ hash, id }) {
+        const entries = await this.raw.getEntries(hash);
+        const metaEnt = entries.find((ent) => ent.id.endsWith(".metadata"));
+        const contentEnt = entries.find((ent) => ent.id.endsWith(".content"));
+        if (metaEnt === undefined) {
+            throw new Error(`couldn't find metadata for hash ${hash}`);
+        }
+        else if (contentEnt === undefined) {
+            throw new Error(`couldn't find content for hash ${hash}`);
+        }
+        const [{ visibleName, lastModified, pinned, parent, lastOpened }, content] = await Promise.all([
+            this.raw.getMetadata(metaEnt.hash),
+            this.raw.getContent(contentEnt.hash),
+        ]);
+        if (content.fileType === undefined) {
+            return {
+                id,
+                hash,
+                visibleName,
+                lastModified,
+                pinned,
+                parent,
+                tags: content.tags,
+                type: "CollectionType",
+            };
+        }
+        else {
+            return {
+                id,
+                hash,
+                visibleName,
+                lastModified,
+                pinned,
+                parent,
+                tags: content.tags,
+                lastOpened: lastOpened ?? "",
+                fileType: content.fileType,
+                type: "DocumentType",
+            };
+        }
+    }
+    /** list all items */
+    async listItems(refresh = false) {
+        const ids = await this.listIds(refresh);
+        return await Promise.all(ids.map((id) => this.#convertEntry(id)));
     }
-    /** list all files */
-    async listFiles({ verify = true } = {}) {
-        const res = await this.#fileRequest();
-        return verification(res, entries, verify);
+    async listIds(refresh = false) {
+        const [hash] = await this.#getRootHash(refresh);
+        const entries = await this.raw.getEntries(hash);
+        return entries.map(({ id, hash }) => ({ id, hash }));
     }
-    /** upload a file */
-    async #uploadFile(parent, visibleName, buffer, contentType, verify) {
-        if (verify && !idReg.test(parent)) {
+    async getContent(hash) {
+        const entries = await this.raw.getEntries(hash);
+        const [cont] = entries.filter((e) => e.id.endsWith(".content"));
+        if (cont === undefined) {
+            throw new Error(`couldn't find contents for hash ${hash}`);
+        }
+        else {
+            return await this.raw.getContent(cont.hash);
+        }
+    }
+    async getMetadata(hash) {
+        const entries = await this.raw.getEntries(hash);
+        const [meta] = entries.filter((e) => e.id.endsWith(".metadata"));
+        if (meta === undefined) {
+            throw new Error(`couldn't find metadata for hash ${hash}`);
+        }
+        else {
+            return await this.raw.getMetadata(meta.hash);
+        }
+    }
+    async getPdf(hash) {
+        const entries = await this.raw.getEntries(hash);
+        const [pdf] = entries.filter((e) => e.id.endsWith(".pdf"));
+        if (pdf === undefined) {
+            throw new Error(`couldn't find pdf for hash ${hash}`);
+        }
+        else {
+            return await this.raw.getHash(pdf.hash);
+        }
+    }
+    async getEpub(hash) {
+        const entries = await this.raw.getEntries(hash);
+        const [epub] = entries.filter((e) => e.id.endsWith(".epub"));
+        if (epub === undefined) {
+            throw new Error(`couldn't find epub for hash ${hash}`);
+        }
+        else {
+            return await this.raw.getHash(epub.hash);
+        }
+    }
+    async getDocument(hash) {
+        const entries = await this.raw.getEntries(hash);
+        const zip = new JSZip();
+        for (const entry of entries) {
+            // TODO if this is .metadata we might want to assert type === "DocumentType"
+            zip.file(entry.id, this.raw.getHash(entry.hash));
+        }
+        return zip.generateAsync({ type: "uint8array" });
+    }
+    async #putFile(visibleName, fileType, buffer, { refresh, parent = "", pinned = false, zoomMode = "bestFit", viewBackgroundFilter, textScale = 1, textAlignment = "justify", fontName = "", coverPageNumber = -1, authors, title, publicationDate, publisher, extraMetadata = {}, lineHeight = -1, margins = 125, orientation = "portrait", tags, }) {
+        if (parent && !idReg.test(parent)) {
             throw new ValidationError(parent, idReg, "parent must be a valid document id");
         }
-        const res = await this.#fileRequest({
-            meta: JSON.stringify({ parent, file_name: visibleName }),
-            method: "POST",
-            contentType,
-            body: buffer,
-        });
-        return verification(res, uploadEntry, verify);
+        const id = uuid4();
+        const now = new Date();
+        const metadata = {
+            parent,
+            pinned,
+            lastModified: (+now).toFixed(),
+            createdTime: (+now).toFixed(),
+            type: "DocumentType",
+            visibleName,
+            lastOpened: "0",
+            lastOpenedPage: 0,
+        };
+        const content = {
+            coverPageNumber,
+            documentMetadata: { authors, title, publicationDate, publisher },
+            extraMetadata,
+            lineHeight,
+            margins,
+            orientation,
+            fileType,
+            formatVersion: 1,
+            tags: tags?.map((name) => ({ name, timestamp: +now })) ?? [],
+            fontName,
+            textAlignment,
+            textScale,
+            zoomMode,
+            viewBackgroundFilter,
+            // NOTE for some reason we need to "fake" the number of pages at 1, and
+            // create "valid" output for that
+            originalPageCount: 1,
+            pageCount: 1,
+            pageTags: [],
+            pages: [uuid4()],
+            redirectionPageMap: [0],
+            sizeInBytes: buffer.length.toFixed(),
+        };
+        // upload raw files, and get root hash
+        const [[contentEntry, uploadContent], [metadataEntry, uploadMetadata], [pagedataEntry, uploadPagedata], [fileEntry, uploadFile], [rootHash, generation],] = await Promise.all([
+            this.raw.putContent(`${id}.content`, content),
+            this.raw.putMetadata(`${id}.metadata`, metadata),
+            // eslint-disable-next-line spellcheck/spell-checker
+            this.raw.putText(`${id}.pagedata`, "\n"),
+            this.raw.putFile(`${id}.${fileType}`, buffer),
+            this.#getRootHash(refresh),
+        ]);
+        // now fetch root entries and upload this file entry
+        const [[collectionEntry, uploadCollection], rootEntries] = await Promise.all([
+            this.raw.putEntries(id, [
+                contentEntry,
+                metadataEntry,
+                pagedataEntry,
+                fileEntry,
+            ]),
+            this.raw.getEntries(rootHash),
+        ]);
+        // now upload a new root entry
+        rootEntries.push(collectionEntry);
+        const [rootEntry, uploadRoot] = await this.raw.putEntries("root", rootEntries);
+        // before updating the root hash, first upload everything
+        await Promise.all([
+            uploadContent,
+            uploadMetadata,
+            uploadPagedata,
+            uploadFile,
+            uploadCollection,
+            uploadRoot,
+        ]);
+        // TODO we could return a full entry here, but we should probably decide
+        // what that should be, e.g. we could return more fields than the standard
+        // entry. Same for putFolder
+        // TODO we should also decide if the api should take hashes or ids...
+        await this.#putRootHash(rootEntry.hash, generation);
+        return { id, hash: collectionEntry.hash };
+    }
+    async putPdf(visibleName, buffer, opts = {}) {
+        return await this.#putFile(visibleName, "pdf", buffer, opts);
+    }
+    async putEpub(visibleName, buffer, opts = {}) {
+        return await this.#putFile(visibleName, "epub", buffer, opts);
     }
     /** create a folder */
-    async createFolder(visibleName, { parent = "", verify = true } = {}) {
-        return await this.#uploadFile(parent, visibleName, new ArrayBuffer(0), "folder", verify);
+    async createFolder(visibleName, { parent = "" } = {}, refresh = false) {
+        if (parent && !idReg.test(parent)) {
+            throw new ValidationError(parent, idReg, "parent must be a valid document id");
+        }
+        const id = uuid4();
+        const now = new Date();
+        const content = {
+            tags: [],
+        };
+        const metadata = {
+            lastModified: (+now).toFixed(),
+            createdTime: (+now).toFixed(),
+            parent,
+            pinned: false,
+            type: "CollectionType",
+            visibleName,
+        };
+        // upload folder contents
+        const [[contentEntry, uploadContent], [metadataEntry, uploadMetadata], [rootHash, generation],] = await Promise.all([
+            this.raw.putContent(`${id}.content`, content),
+            this.raw.putMetadata(`${id}.metadata`, metadata),
+            this.#getRootHash(refresh),
+        ]);
+        // now fetch root entries and upload this file entry
+        const [[collectionEntry, uploadCollection], rootEntries] = await Promise.all([
+            this.raw.putEntries(id, [contentEntry, metadataEntry]),
+            this.raw.getEntries(rootHash),
+        ]);
+        // now upload a new root entry
+        rootEntries.push(collectionEntry);
+        const [rootEntry, uploadRoot] = await this.raw.putEntries("root", rootEntries);
+        // before updating the root hash, first upload everything
+        await Promise.all([
+            uploadContent,
+            uploadMetadata,
+            uploadCollection,
+            uploadRoot,
+        ]);
+        // put root hash and return
+        await this.#putRootHash(rootEntry.hash, generation);
+        return { id, hash: collectionEntry.hash };
     }
     /** upload an epub */
-    async uploadEpub(visibleName, buffer, { parent = "", verify = true } = {}) {
-        return await this.#uploadFile(parent, visibleName, buffer, "application/epub+zip", verify);
+    async uploadEpub(visibleName, buffer, opts = {}) {
+        return await this.putEpub(visibleName, buffer, opts);
     }
     /** upload a pdf */
-    async uploadPdf(visibleName, buffer, { parent = "", verify = true } = {}) {
-        return await this.#uploadFile(parent, visibleName, buffer, "application/pdf", verify);
+    async uploadPdf(visibleName, buffer, opts = {}) {
+        return await this.putPdf(visibleName, buffer, opts);
     }
-    async #modify(hash, properties, verify) {
-        if (verify && !hashReg.test(hash)) {
-            throw new ValidationError(hash, hashReg, "hash to modify was not a valid hash");
+    async #editMetaRaw(id, hash, update) {
+        const entries = await this.raw.getEntries(hash);
+        const metaInd = entries.findIndex((ent) => ent.id.endsWith(".metadata"));
+        const metaEntry = entries[metaInd];
+        if (metaEntry === undefined) {
+            throw new Error("internal error: couldn't find metadata in entry hash");
         }
-        // this does not allow setting pinned, although I don't know why
-        const res = await this.#fileRequest({
-            hash,
-            body: JSON.stringify(properties),
-            method: "PATCH",
-        });
-        return verification(res, hashEntry, verify);
+        const meta = await this.raw.getMetadata(metaEntry.hash);
+        Object.assign(meta, update);
+        const [newMetaEntry, uploadMeta] = await this.raw.putMetadata(metaEntry.id, meta);
+        entries[metaInd] = newMetaEntry;
+        const [result, uploadentries] = await this.raw.putEntries(id, entries);
+        const upload = Promise.all([uploadMeta, uploadentries]).then(() => { });
+        return [result, upload];
+    }
+    async #editMeta(hash, update, refresh = false) {
+        const [rootHash, generation] = await this.#getRootHash(refresh);
+        const entries = await this.raw.getEntries(rootHash);
+        const hashInd = entries.findIndex((ent) => ent.hash === hash);
+        const hashEnt = entries[hashInd];
+        if (hashEnt === undefined) {
+            throw new HashNotFoundError(hash);
+        }
+        const [newEnt, uploadEnt] = await this.#editMetaRaw(hashEnt.id, hash, update);
+        entries[hashInd] = newEnt;
+        const [rootEntry, uploadRoot] = await this.raw.putEntries("root", entries);
+        await Promise.all([uploadEnt, uploadRoot]);
+        await this.#putRootHash(rootEntry.hash, generation);
+        return { hash: newEnt.hash };
     }
     /** move an entry */
-    async move(hash, parent, { verify = true } = {}) {
-        if (verify && !idReg.test(parent)) {
+    async move(hash, parent, refresh = false) {
+        if (!idReg.test(parent)) {
             throw new ValidationError(parent, idReg, "parent must be a valid document id");
         }
-        return await this.#modify(hash, { parent }, verify);
+        return await this.#editMeta(hash, { parent }, refresh);
     }
     /** delete an entry */
-    async delete(hash, opts = {}) {
-        return await this.move(hash, "trash", opts);
+    async delete(hash, refresh = false) {
+        return await this.move(hash, "trash", refresh);
     }
     /** rename an entry */
-    async rename(hash, visibleName, { verify = true } = {}) {
-        return await this.#modify(hash, { file_name: visibleName }, verify);
-    }
-    /** bulk modify hashes */
-    async #bulkModify(hashes, properties, verify) {
-        if (verify) {
-            const invalidHashes = hashes.filter((hash) => !hashReg.test(hash));
-            if (invalidHashes.length) {
-                throw new ValidationError(hashes.join(", "), hashReg, "hashes to modify were not a valid hashes");
-            }
-        }
-        // this does not allow setting pinned, although I don't know why
-        const res = await this.#fileRequest({
-            body: JSON.stringify({
-                updates: properties,
-                hashes,
-            }),
-            method: "PATCH",
-        });
-        return verification(res, hashesEntry, verify);
+    async rename(hash, visibleName, refresh = false) {
+        return await this.#editMeta(hash, { visibleName }, refresh);
     }
     /** move many hashes */
-    async bulkMove(hashes, parent, { verify = true } = {}) {
-        if (verify && !idReg.test(parent)) {
+    async bulkMove(hashes, parent, refresh = false) {
+        if (!idReg.test(parent)) {
             throw new ValidationError(parent, idReg, "parent must be a valid document id");
         }
-        return await this.#bulkModify(hashes, { parent }, verify);
+        const [rootHash, generation] = await this.#getRootHash(refresh);
+        const entries = await this.raw.getEntries(rootHash);
+        const hashSet = new Set(hashes);
+        const toUpdate = [];
+        const newEntries = [];
+        for (const entry of entries) {
+            const part = hashSet.has(entry.hash) ? toUpdate : newEntries;
+            part.push(entry);
+        }
+        const resolved = await Promise.all(toUpdate.map(({ id, hash }) => this.#editMetaRaw(id, hash, { parent })));
+        const uploads = [];
+        const result = {};
+        for (const [i, [newEnt, upload]] of resolved.entries()) {
+            newEntries.push(newEnt);
+            uploads.push(upload);
+            result[toUpdate[i].hash] = newEnt.hash;
+        }
+        const [rootEntry, uploadRoot] = await this.raw.putEntries("root", newEntries);
+        uploads.push(uploadRoot);
+        await Promise.all(uploads);
+        await this.#putRootHash(rootEntry.hash, generation);
+        return { hashes: result };
     }
     /** delete many hashes */
-    async bulkDelete(hashes, opts = {}) {
-        return await this.bulkMove(hashes, "trash", opts);
+    async bulkDelete(hashes, refresh = false) {
+        return await this.bulkMove(hashes, "trash", refresh);
+    }
+    /** dump the raw cache */
+    dumpCache() {
+        return this.raw.dumpCache();
+    }
+    async pruneCache(refresh) {
+        const [rootHash] = await this.#getRootHash(refresh);
+        // the keys to delete, we'll drop every key we can currently reach
+        const toDelete = new Set(this.#cache.keys());
+        // bfs through entries (to semi-optimize promise waiting, although this
+        // should only go one step) to track all hashes encountered
+        // NOTE that we could increase the cache in this process, or it's possible
+        // for other calls to increase the cache with misc values.
+        let entries = [await this.raw.getEntries(rootHash)];
+        let nextEntries = [];
+        while (entries.length) {
+            for (const entryList of entries) {
+                for (const { hash, type } of entryList) {
+                    toDelete.add(hash);
+                    if (type === 80000000) {
+                        nextEntries.push(this.raw.getEntries(hash));
+                    }
+                }
+            }
+            entries = await Promise.all(nextEntries);
+            nextEntries = [];
+        }
+        for (const key of toDelete) {
+            this.#cache.delete(key);
+        }
+    }
+    // finally remove any values we had in the cache initially, but couldn't reach
+    clearCache() {
+        this.raw.clearCache();
     }
 }
+const cached = values(nullable(string()));
 /**
  * create an instance of the api
  *
@@ -297,7 +952,7 @@ class Remarkable {
  *    registered. Create one with {@link register}.
  * @returns an api instance
  */
-export async function remarkable(deviceToken, { fetch = globalThis.fetch, authHost = AUTH_HOST, syncHost = SYNC_HOST, } = {}) {
+export async function remarkable(deviceToken, { authHost = AUTH_HOST, rawHost = RAW_HOST, cache, maxCacheSize = Infinity, } = {}) {
     const resp = await fetch(`${authHost}/token/json/2/user/new`, {
         method: "POST",
         headers: {
@@ -308,5 +963,15 @@ export async function remarkable(deviceToken, { fetch = globalThis.fetch, authHo
         throw new Error(`couldn't fetch auth token: ${resp.statusText}`);
     }
     const userToken = await resp.text();
-    return new Remarkable(userToken, fetch, syncHost);
+    const initCache = JSON.parse(cache ?? "{}");
+    if (cached.guard(initCache)) {
+        const entries = Object.entries(initCache);
+        const cache = maxCacheSize === Infinity
+            ? new Map(entries)
+            : new LruCache(maxCacheSize, entries);
+        return new Remarkable(userToken, rawHost, cache);
+    }
+    else {
+        throw new Error("cache was not a valid cache (json string mapping); your cache must be corrupted somehow. Either initialize remarkable without a cache, or fix its format.");
+    }
 }