styled-map-package-api 5.0.0-pre.0

package/dist/writer.js

@@ -0,0 +1,452 @@
+import { validateStyleMin, migrate } from "@maplibre/maplibre-gl-style-spec";
+import { bbox } from "@turf/bbox";
+import { EventEmitter } from "events";
+import { excludeKeys } from "filter-obj";
+import { ZipWriter } from "zip-writer";
+import { getTileFormatFromStream } from "./utils/file-formats.js";
+import { MAX_BOUNDS, tileToBBox, unionBBox } from "./utils/geo.js";
+import { clone } from "./utils/misc.js";
+import { writeStreamFromAsync } from "./utils/streams.js";
+import { replaceFontStacks } from "./utils/style.js";
+import {
+  FONTS_FOLDER,
+  FORMAT_VERSION,
+  getGlyphFilename,
+  getSpriteFilename,
+  getSpriteUri,
+  getTileFilename,
+  getTileUri,
+  GLYPH_URI,
+  SOURCES_FOLDER,
+  STYLE_FILE,
+  VERSION_FILE
+} from "./utils/templates.js";
+const SUPPORTED_SOURCE_TYPES = (
+  /** @type {const} */
+  [
+    "raster",
+    "vector",
+    "geojson"
+  ]
+);
+class Writer extends EventEmitter {
+  #zipWriter = new ZipWriter();
+  /** @type {Set<string>} */
+  #addedFiles = /* @__PURE__ */ new Set();
+  /** @type {Set<string>} */
+  #fonts = /* @__PURE__ */ new Set();
+  /** @type {Set<string>} */
+  #addedSpriteIds = /* @__PURE__ */ new Set();
+  /** @type {Map<string, SourceInfo>} */
+  #sources = /* @__PURE__ */ new Map();
+  /** @type {StyleSpecification} */
+  #style;
+  /** @type {ReadableStream<Uint8Array>} */
+  #outputStream;
+  /** @type {ReadableStreamDefaultController<Uint8Array>} */
+  #outputController;
+  static SUPPORTED_SOURCE_TYPES = SUPPORTED_SOURCE_TYPES;
+  /**
+   * @param {any} style A v7 or v8 MapLibre style. v7 styles will be migrated to
+   * v8. (There are currently no typescript declarations for v7 styles, hence
+   * this is typed as `any` and validated internally)
+   */
+  constructor(style) {
+    super();
+    if (!style || !("version" in style)) {
+      throw new Error("Invalid style");
+    }
+    if (style.version !== 7 && style.version !== 8) {
+      throw new Error(`Invalid style: Unsupported version v${style.version}`);
+    }
+    if (!Array.isArray(style.layers)) {
+      throw new Error("Invalid style: missing layers property");
+    }
+    const styleCopy = clone(style);
+    migrate(styleCopy);
+    const errors = validateStyleMin(styleCopy);
+    if (errors.length) {
+      throw new AggregateError(errors, "Invalid style");
+    }
+    this.#style = styleCopy;
+    for (const [sourceId, source] of Object.entries(this.#style.sources)) {
+      if (source.type !== "geojson") continue;
+      this.#addSource(sourceId, source);
+    }
+    const zipReader = this.#zipWriter.readable.getReader();
+    let outputController;
+    this.#outputStream = new ReadableStream({
+      start(controller) {
+        outputController = controller;
+      },
+      async pull(controller) {
+        try {
+          const { done, value } = await zipReader.read();
+          if (done) {
+            controller.close();
+          } else {
+            controller.enqueue(
+              /** @type {Uint8Array} */
+              value
+            );
+          }
+        } catch (err) {
+          controller.error(err);
+        }
+      },
+      cancel(reason) {
+        zipReader.cancel(reason);
+      }
+    });
+    this.#outputController = outputController;
+  }
+  /**
+   * @returns {ReadableStream<Uint8Array>} Readable stream of the styled map package
+   */
+  get outputStream() {
+    return this.#outputStream;
+  }
+  /**
+   * Abort the output stream with an error. Call this if an error occurs during
+   * writing to propagate the error to consumers of `outputStream`.
+   *
+   * @param {Error} reason
+   */
+  abort(reason) {
+    this.#outputController.error(reason);
+  }
+  #getBounds() {
+    let bounds;
+    let maxzoom = 0;
+    for (const { source } of this.#sources.values()) {
+      if (source.type === "geojson") {
+        if (isEmptyFeatureCollection(source.data)) continue;
+        const bbox2 = get2DBBox(source.data.bbox);
+        bounds = bounds ? unionBBox([bounds, bbox2]) : [...bbox2];
+      } else {
+        if (source.maxzoom < maxzoom) continue;
+        if (source.maxzoom === maxzoom) {
+          bounds = bounds ? unionBBox([bounds, source.bounds]) : source.bounds;
+        } else {
+          bounds = source.bounds;
+          maxzoom = source.maxzoom;
+        }
+      }
+    }
+    return bounds;
+  }
+  #getMaxZoom() {
+    let maxzoom = 0;
+    for (const { source } of this.#sources.values()) {
+      const sourceMaxzoom = (
+        // For GeoJSON sources, the maxzoom is 16 unless otherwise set
+        source.type === "geojson" ? source.maxzoom || 16 : source.maxzoom
+      );
+      maxzoom = Math.max(maxzoom, sourceMaxzoom);
+    }
+    return maxzoom;
+  }
+  /**
+   * Add a source definition to the styled map package
+   *
+   * @param {string} sourceId
+   * @param {InputSource} source
+   * @returns {SourceInfo}
+   */
+  #addSource(sourceId, source) {
+    const encodedSourceId = encodeSourceId(this.#sources.size);
+    const tileSourceOverrides = {
+      minzoom: 0,
+      maxzoom: 0,
+      bounds: (
+        /** @type {import('./utils/geo.js').BBox} */
+        [...MAX_BOUNDS]
+      ),
+      tiles: (
+        /** @type {string[]} */
+        []
+      )
+    };
+    let smpSource;
+    switch (source.type) {
+      case "raster":
+      case "vector":
+        smpSource = {
+          ...excludeKeys(source, ["tiles", "url"]),
+          ...tileSourceOverrides
+        };
+        break;
+      case "geojson":
+        smpSource = {
+          ...source,
+          maxzoom: 0,
+          data: typeof source.data !== "string" ? (
+            // Add a bbox property to the GeoJSON data if it doesn't already have one
+            { ...source.data, bbox: source.data.bbox || bbox(source.data) }
+          ) : (
+            // If GeoJSON data is referenced by a URL, start with an empty FeatureCollection
+            {
+              type: "FeatureCollection",
+              features: [],
+              bbox: [0, 0, 0, 0]
+            }
+          )
+        };
+        break;
+    }
+    const sourceInfo = {
+      source: smpSource,
+      encodedSourceId
+    };
+    this.#sources.set(sourceId, sourceInfo);
+    return sourceInfo;
+  }
+  /**
+   * Add a tile to the styled map package
+   *
+   * @param {Source} tileData
+   * @param {TileInfo} opts
+   */
+  async addTile(tileData, { z, x, y, sourceId, format }) {
+    let sourceInfo = this.#sources.get(sourceId);
+    if (!sourceInfo) {
+      const source2 = this.#style.sources[sourceId];
+      if (!source2) {
+        throw new Error(`Source not referenced in style.json: ${sourceId}`);
+      }
+      if (source2.type !== "raster" && source2.type !== "vector") {
+        throw new Error(`Unsupported source type: ${source2.type}`);
+      }
+      sourceInfo = this.#addSource(sourceId, source2);
+    }
+    const { source, encodedSourceId } = sourceInfo;
+    if (source.type !== "raster" && source.type !== "vector") {
+      throw new Error(`Unsupported source type: ${source.type}`);
+    }
+    if (!format) {
+      ;
+      [format, tileData] = await getTileFormatFromStream(toWebStream(tileData));
+    }
+    if (!sourceInfo.format) {
+      sourceInfo.format = format;
+    } else if (sourceInfo.format !== format) {
+      throw new Error(
+        `Tile format mismatch for source ${sourceId}: expected ${sourceInfo.format}, got ${format}`
+      );
+    }
+    const bbox2 = tileToBBox({ z, x, y });
+    if (z > source.maxzoom) {
+      source.maxzoom = z;
+      source.bounds = bbox2;
+    } else if (z === source.maxzoom) {
+      source.bounds = unionBBox([source.bounds, bbox2]);
+    }
+    const name = getTileFilename({ sourceId: encodedSourceId, z, x, y, format });
+    return this.#append(tileData, { name, store: true });
+  }
+  /**
+   * Create a write stream for adding tiles to the styled map package
+   *
+   * @param {object} opts
+   * @param {number} [opts.concurrency=16] The number of concurrent writes
+   *
+   * @returns
+   */
+  createTileWriteStream({ concurrency = 16 } = {}) {
+    return writeStreamFromAsync(this.addTile.bind(this), { concurrency });
+  }
+  /**
+   * Add a sprite to the styled map package
+   *
+   * @param {object} options
+   * @param {Source} options.json
+   * @param {Source} options.png
+   * @param {number} [options.pixelRatio]
+   * @param {string} [options.id='default']
+   * @returns {Promise<void>}
+   */
+  async addSprite({ json, png, pixelRatio = 1, id = "default" }) {
+    this.#addedSpriteIds.add(id);
+    const jsonName = getSpriteFilename({ id, pixelRatio, ext: ".json" });
+    const pngName = getSpriteFilename({ id, pixelRatio, ext: ".png" });
+    await Promise.all([
+      this.#append(json, { name: jsonName }),
+      this.#append(png, { name: pngName })
+    ]);
+  }
+  /**
+   * Add glyphs to the styled map package
+   *
+   * @param {Source} glyphData
+   * @param {GlyphInfo} glyphInfo
+   * @returns {Promise<void>}
+   */
+  addGlyphs(glyphData, { font: fontName, range }) {
+    this.#fonts.add(fontName);
+    const name = getGlyphFilename({ fontstack: fontName, range });
+    return this.#append(glyphData, { name });
+  }
+  /**
+   * Create a write stream for adding glyphs to the styled map package
+   *
+   * @param {object} opts
+   * @param {number} [opts.concurrency=16] The number of concurrent writes
+   * @returns
+   */
+  createGlyphWriteStream({ concurrency = 16 } = {}) {
+    return writeStreamFromAsync(this.addGlyphs.bind(this), { concurrency });
+  }
+  /**
+   * Finalize the styled map package and write the style to the archive.
+   * This method must be called to complete the archive.
+   * You must wait for your destination write stream to 'finish' before using the output.
+   */
+  async finish() {
+    await this.#append(FORMAT_VERSION, { name: VERSION_FILE });
+    this.#prepareStyle();
+    const style = JSON.stringify(this.#style);
+    await this.#append(style, { name: STYLE_FILE });
+    const entries = await this.#zipWriter.entries();
+    const sortedEntries = sortEntries(entries);
+    await this.#zipWriter.finalize({ entries: sortedEntries });
+  }
+  /**
+   * Mutates the style object to prepare it for writing to the archive.
+   * Deterministic: can be run more than once with the same result.
+   */
+  #prepareStyle() {
+    if (this.#sources.size === 0) {
+      throw new Error("Missing sources: add at least one source");
+    }
+    if (this.#style.glyphs && this.#fonts.size === 0) {
+      throw new Error(
+        "Missing fonts: style references glyphs but no fonts added"
+      );
+    }
+    replaceFontStacks(this.#style, [...this.#fonts]);
+    if (this.#style.glyphs) {
+      this.#style.glyphs = GLYPH_URI;
+    }
+    if (typeof this.#style.sprite === "string") {
+      if (!this.#addedSpriteIds.has("default")) {
+        throw new Error(
+          "Missing sprite: style references sprite but none added"
+        );
+      }
+      this.#style.sprite = getSpriteUri();
+    } else if (Array.isArray(this.#style.sprite)) {
+      this.#style.sprite = this.#style.sprite.map(({ id }) => {
+        if (!this.#addedSpriteIds.has(id)) {
+          throw new Error(
+            `Missing sprite: style references sprite ${id} but none added`
+          );
+        }
+        return { id, url: getSpriteUri(id) };
+      });
+    }
+    this.#style.sources = {};
+    for (const [sourceId, { source, encodedSourceId, format = "mvt" }] of this.#sources) {
+      if (source.type === "geojson" && isEmptyFeatureCollection(source.data)) {
+        continue;
+      }
+      this.#style.sources[sourceId] = source;
+      if (!("tiles" in source)) continue;
+      source.tiles = [getTileUri({ sourceId: encodedSourceId, format })];
+    }
+    this.#style.layers = this.#style.layers.filter(
+      (layer) => !("source" in layer) || !!this.#style.sources[layer.source]
+    );
+    const metadata = this.#style.metadata || (this.#style.metadata = {});
+    const bounds = this.#getBounds();
+    if (bounds) {
+      metadata["smp:bounds"] = bounds;
+      const [w, s, e, n] = bounds;
+      this.#style.center = [w + (e - w) / 2, s + (n - s) / 2];
+    }
+    metadata["smp:maxzoom"] = this.#getMaxZoom();
+    metadata["smp:sourceFolders"] = {};
+    for (const [sourceId, { encodedSourceId }] of this.#sources) {
+      metadata["smp:sourceFolders"][sourceId] = encodedSourceId;
+    }
+    this.#style.zoom = Math.max(0, this.#getMaxZoom() - 2);
+  }
+  /**
+   *
+   * @param {Source} source
+   * @param {{ name: string, store?: boolean }} options
+   * @returns {Promise<void>}
+   */
+  async #append(source, { name, store = false }) {
+    if (this.#addedFiles.has(name)) {
+      throw new Error(`${name} already added`);
+    }
+    this.#addedFiles.add(name);
+    const readable = toWebStream(source);
+    await this.#zipWriter.addEntry({ readable, name, store });
+  }
+}
+function encodeSourceId(sourceIndex) {
+  return sourceIndex.toString(36);
+}
+function toWebStream(source) {
+  if (typeof source === "string") {
+    const bytes = new TextEncoder().encode(source);
+    return new ReadableStream({
+      start(controller) {
+        controller.enqueue(bytes);
+        controller.close();
+      }
+    });
+  }
+  if (source instanceof Uint8Array) {
+    return new ReadableStream({
+      start(controller) {
+        controller.enqueue(
+          new Uint8Array(source.buffer, source.byteOffset, source.byteLength)
+        );
+        controller.close();
+      }
+    });
+  }
+  return (
+    /** @type {ReadableStream<Uint8Array>} */
+    source
+  );
+}
+function isEmptyFeatureCollection(data) {
+  return data.type === "FeatureCollection" && data.features.length === 0;
+}
+function get2DBBox(bbox2) {
+  if (bbox2.length === 4) return bbox2;
+  return [bbox2[0], bbox2[1], bbox2[3], bbox2[4]];
+}
+function sortEntries(entries) {
+  return [...entries].sort((a, b) => {
+    if (a.name === VERSION_FILE) return -1;
+    if (b.name === VERSION_FILE) return 1;
+    if (a.name === STYLE_FILE) return -1;
+    if (b.name === STYLE_FILE) return 1;
+    const foldersA = a.name.split("/");
+    const foldersB = b.name.split("/");
+    const aIsFirst = foldersA[0] === FONTS_FOLDER && foldersA[2] === "0-255.pbf.gz";
+    const bIsFirst = foldersB[0] === FONTS_FOLDER && foldersB[2] === "0-255.pbf.gz";
+    if (aIsFirst && bIsFirst)
+      return a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
+    if (aIsFirst) return -1;
+    if (bIsFirst) return 1;
+    if (foldersA[0] === SOURCES_FOLDER && foldersB[0] !== SOURCES_FOLDER)
+      return -1;
+    if (foldersB[0] === SOURCES_FOLDER && foldersA[0] !== SOURCES_FOLDER)
+      return 1;
+    if (foldersA[0] === SOURCES_FOLDER && foldersB[0] === SOURCES_FOLDER) {
+      const zoomA = +foldersA[2];
+      const zoomB = +foldersB[2];
+      return zoomA - zoomB;
+    }
+    return 0;
+  });
+}
+export {
+  SUPPORTED_SOURCE_TYPES,
+  Writer
+};

package/package.json

@@ -0,0 +1,161 @@
+{
+  "name": "styled-map-package-api",
+  "version": "5.0.0-pre.0",
+  "description": "JavaScript API for reading, writing, and serving Styled Map Package (.smp) files",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    },
+    "./reader": {
+      "import": {
+        "types": "./dist/reader.d.ts",
+        "default": "./dist/reader.js"
+      },
+      "require": {
+        "types": "./dist/reader.d.cts",
+        "default": "./dist/reader.cjs"
+      }
+    },
+    "./writer": {
+      "import": {
+        "types": "./dist/writer.d.ts",
+        "default": "./dist/writer.js"
+      },
+      "require": {
+        "types": "./dist/writer.d.cts",
+        "default": "./dist/writer.cjs"
+      }
+    },
+    "./server": {
+      "import": {
+        "types": "./dist/server.d.ts",
+        "default": "./dist/server.js"
+      },
+      "require": {
+        "types": "./dist/server.d.cts",
+        "default": "./dist/server.cjs"
+      }
+    },
+    "./style-downloader": {
+      "import": {
+        "types": "./dist/style-downloader.d.ts",
+        "default": "./dist/style-downloader.js"
+      },
+      "require": {
+        "types": "./dist/style-downloader.d.cts",
+        "default": "./dist/style-downloader.cjs"
+      }
+    },
+    "./tile-downloader": {
+      "import": {
+        "types": "./dist/tile-downloader.d.ts",
+        "default": "./dist/tile-downloader.js"
+      },
+      "require": {
+        "types": "./dist/tile-downloader.d.cts",
+        "default": "./dist/tile-downloader.cjs"
+      }
+    },
+    "./download": {
+      "import": {
+        "types": "./dist/download.d.ts",
+        "default": "./dist/download.js"
+      },
+      "require": {
+        "types": "./dist/download.d.cts",
+        "default": "./dist/download.cjs"
+      }
+    },
+    "./from-mbtiles": {
+      "import": {
+        "types": "./dist/from-mbtiles.d.ts",
+        "default": "./dist/from-mbtiles.js"
+      },
+      "require": {
+        "types": "./dist/from-mbtiles.d.cts",
+        "default": "./dist/from-mbtiles.cjs"
+      }
+    },
+    "./utils/mapbox": {
+      "import": {
+        "types": "./dist/utils/mapbox.d.ts",
+        "default": "./dist/utils/mapbox.js"
+      },
+      "require": {
+        "types": "./dist/utils/mapbox.d.cts",
+        "default": "./dist/utils/mapbox.cjs"
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "test": "npm run build && npm run types && vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "build": "tsup",
+    "types": "tsc",
+    "bench": "vitest bench --project node",
+    "prepack": "npm run build"
+  },
+  "keywords": [],
+  "license": "MIT",
+  "dependencies": {
+    "@gmaclennan/zip-reader": "^1.0.0-pre.2",
+    "@mapbox/sphericalmercator": "^1.2.0",
+    "@maplibre/maplibre-gl-style-spec": "^20.3.1",
+    "@placemarkio/check-geojson": "^0.1.12",
+    "@turf/bbox": "^7.2.0",
+    "@turf/helpers": "^7.2.0",
+    "filter-obj": "^6.1.0",
+    "itty-router": "^5.0.22",
+    "ky": "^1.7.5",
+    "map-obj": "^5.0.2",
+    "mbtiles-reader": "^1.0.0",
+    "p-limit": "^6.2.0",
+    "readable-stream": "^4.7.0",
+    "yocto-queue": "^1.1.1",
+    "zip-writer": "^2.2.0"
+  },
+  "devDependencies": {
+    "@commander-js/extra-typings": "^12.1.0",
+    "@types/geojson": "^7946.0.16",
+    "@types/http-errors": "^2.0.4",
+    "@types/mapbox__sphericalmercator": "^1.2.3",
+    "@types/node": "^18.19.130",
+    "@types/readable-stream": "^4.0.18",
+    "@vitest/browser": "^3.2.4",
+    "@vitest/coverage-v8": "^3.2.4",
+    "@whatwg-node/server": "^0.10.17",
+    "esbuild-fix-imports-plugin": "^1.0.20",
+    "execa": "^9.4.0",
+    "fast-png": "^6.3.0",
+    "pixel-stream": "^1.0.3",
+    "playwright": "^1.50.1",
+    "sharp": "^0.33.5",
+    "tempy": "^3.1.0",
+    "tsup": "^8.4.0",
+    "type-fest": "^4.35.0",
+    "typescript": "^5.9.3",
+    "vite": "^6.4.1",
+    "vitest": "^3.2.4"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/digidem/styled-map-package.git",
+    "directory": "packages/api"
+  }
+}