styled-map-package 1.0.0

package/lib/types.ts

@@ -0,0 +1,104 @@
+import type {
+  SourceSpecification,
+  StyleSpecification,
+  ValidationError,
+  GeoJSONSourceSpecification,
+  VectorSourceSpecification,
+  RasterSourceSpecification,
+  RasterDEMSourceSpecification,
+} from '@maplibre/maplibre-gl-style-spec'
+import type { GeoJSON, BBox } from 'geojson'
+import type { Readable } from 'stream'
+import type { Except, SetRequired, Simplify } from 'type-fest'
+
+import { SUPPORTED_SOURCE_TYPES } from './writer.js'
+
+export type InputSource = Extract<
+  SourceSpecification,
+  { type: (typeof SUPPORTED_SOURCE_TYPES)[number] }
+>
+type TransformInlinedSource<T extends SourceSpecification> =
+  T extends GeoJSONSourceSpecification
+    ? OmitUnion<T, 'data'> & { data: GeoJSON }
+    : T extends
+          | VectorSourceSpecification
+          | RasterSourceSpecification
+          | RasterDEMSourceSpecification
+      ? SetRequired<OmitUnion<T, 'url'>, 'tiles'>
+      : T
+/**
+ * This is a slightly stricter version of SourceSpecification that requires
+ * sources to be inlined (e.g. no urls to TileJSON or GeoJSON files).
+ */
+export type InlinedSource = TransformInlinedSource<SourceSpecification>
+type SupportedInlinedSource = Extract<
+  InlinedSource,
+  { type: (typeof SUPPORTED_SOURCE_TYPES)[number] }
+>
+/**
+ * This is a slightly stricter version of StyleSpecification that requires
+ * sources to be inlined (e.g. no urls to TileJSON or GeoJSON files).
+ */
+export type StyleInlinedSources = Omit<StyleSpecification, 'sources'> & {
+  sources: {
+    [_: string]: InlinedSource
+  }
+}
+
+export type SMPSource = TransformSMPInputSource<SupportedInlinedSource>
+/**
+ * This is a slightly stricter version of StyleSpecification that is provided in
+ * a Styled Map Package. Tile sources must have tile URLs inlined (they cannot
+ * refer to a TileJSON url), and they must have bounds, minzoom, and maxzoom.
+ * GeoJSON sources must have inlined GeoJSON (not a URL to a GeoJSON file).
+ */
+export type SMPStyle = TransformSMPStyle<StyleSpecification>
+
+export type TransformSMPInputSource<T extends SupportedInlinedSource> =
+  T extends GeoJSONSourceSpecification
+    ? T & { data: { bbox: BBox } }
+    : T extends RasterSourceSpecification | VectorSourceSpecification
+      ? SetRequired<T, 'bounds' | 'minzoom' | 'maxzoom'>
+      : T
+
+type TransformSMPStyle<T extends StyleSpecification> = Omit<T, 'sources'> & {
+  metadata: {
+    'smp:bounds': [number, number, number, number]
+    'smp:maxzoom': 0
+    'smp:sourceFolders': { [_: string]: string }
+  }
+  sources: {
+    [_: string]: SMPSource
+  }
+}
+
+export interface ValidateStyle {
+  (style: unknown): style is StyleSpecification
+  errors: Array<ValidationError>
+}
+
+export interface DownloadStream extends Readable {
+  iterator(
+    ...args: Parameters<Readable['iterator']>
+  ): AsyncIterableIterator<Buffer>
+  [Symbol.asyncIterator](): AsyncIterableIterator<Buffer>
+}
+
+export type RequiredUnion<T> = T extends any ? Required<T> : never
+export type OmitUnion<T, K extends keyof any> = T extends unknown
+  ? Omit<T, K>
+  : never
+
+type SetRequiredIfPresent<
+  BaseType,
+  Keys extends keyof any,
+> = BaseType extends unknown
+  ? Keys extends keyof BaseType
+    ? Simplify<
+        // Pick just the keys that are optional from the base type.
+        Except<BaseType, Keys> &
+          // Pick the keys that should be required from the base type and make them required.
+          Required<Pick<BaseType, Keys>>
+      >
+    : never
+  : never

package/lib/utils/fetch.js

@@ -0,0 +1,100 @@
+import ky from 'ky'
+import { pEvent } from 'p-event'
+import pLimit from 'p-limit'
+
+import { fromWebReadableStream, ProgressStream } from './streams.js'
+
+/**
+ * @typedef {object} DownloadResponse
+ * @property {import('stream').Readable} body Node ReadableStream of the response body
+ * @property {string | null} mimeType Content mime-type (from http content-type header)
+ * @property {number | null} contentLength Content length in bytes (from http content-length header)
+ */
+
+/**
+ * A wrapper for fetch that limits the number of concurrent downloads.
+ */
+export class FetchQueue {
+  /** @type {import('p-limit').LimitFunction} */
+  #limit
+  /** @param {number} concurrency */
+  constructor(concurrency) {
+    this.#limit = pLimit(concurrency)
+  }
+
+  get activeCount() {
+    return this.#limit.activeCount
+  }
+
+  /**
+   * Fetch a URL, limiting the number of concurrent downloads. Resolves with a
+   * `DownloadResponse`, which is a parsed from the Fetch `Response` objects,
+   * with `body` as a Node readable stream, and the MIME type and content length
+   * of the response.
+   *
+   * NB: The response body stream must be consumed to the end, otherwise the
+   * queue will never be emptied.
+   *
+   * @param {string} url
+   * @param {{ onprogress?: import('./streams.js').ProgressCallback }} opts
+   * @returns {Promise<DownloadResponse>}
+   */
+  fetch(url, { onprogress } = {}) {
+    // This is wrapped like this so that pLimit limits concurrent `fetchStream`
+    // calls, which only resolve when the body is completely downloaded, but
+    // this method will return a response as soon as it is available. NB: If the
+    // body of a response is never "consumed" (e.g. by reading it to the end),
+    // the fetchStream function will never resolve, and the limit will never be
+    // released.
+    return new Promise((resolveResponse, rejectResponse) => {
+      this.#limit(fetchStream, {
+        url,
+        onresponse: resolveResponse,
+        onerror: rejectResponse,
+        onprogress,
+      })
+    })
+  }
+}
+
+/**
+ * This will resolve when the download is complete, regardless of success or
+ * failure, but a readable stream is available before download via the
+ * onReadStream param. This strange function signature is used for limiting the
+ * number of simultaneous downloads, but still being able to expose the Response
+ * as soon as it is available. This is implmented this way to avoid creating
+ * unnecessary closures, which is important here because we can have thousands
+ * of tile requests.
+ *
+ * @param {object} opts
+ * @param {string} opts.url
+ * @param {(response: DownloadResponse) => void} opts.onresponse
+ * @param {(err: Error) => void} opts.onerror
+ * @param {import('./streams.js').ProgressCallback} [opts.onprogress]
+ * @returns {Promise<void>}
+ */
+async function fetchStream({ url, onresponse, onerror, onprogress }) {
+  try {
+    const response = await ky(url, { retry: 3 })
+    if (!response.body) {
+      throw new Error('No body in response')
+    }
+    const body = fromWebReadableStream(response.body)
+    const contentType = response.headers.get('content-type')
+    const mimeType =
+      typeof contentType === 'string' ? contentType.split(';')[0] : null
+    const contentLengthHeader = response.headers.get('content-length')
+    const contentLength =
+      contentLengthHeader === null ? null : parseInt(contentLengthHeader, 10)
+    onresponse({
+      body: onprogress ? body.pipe(new ProgressStream({ onprogress })) : body,
+      mimeType,
+      contentLength,
+    })
+    // Wait for the read stream to end before resolving this function, so that
+    // we limit concurrent downloads
+    await pEvent(body, 'end')
+  } catch (err) {
+    onerror(err instanceof Error ? err : new Error('Unknown error'))
+  }
+}

package/lib/utils/file-formats.js

@@ -0,0 +1,85 @@
+import BufferPeerStream from 'buffer-peek-stream'
+
+import { hasOwn } from './misc.js'
+
+const peek = BufferPeerStream.promise
+
+const MAGIC_BYTES = /** @type {const} */ ({
+  png: [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a],
+  jpg: [0xff, 0xd8, 0xff],
+  // eslint-disable-next-line no-sparse-arrays
+  webp: [0x52, 0x49, 0x46, 0x46, , , , , 0x57, 0x45, 0x42, 0x50],
+  // Include the compression-type byte, which is always 0x08 (DEFLATE) for gzip
+  gz: [0x1f, 0x8b, 0x08],
+})
+
+const MIME_TYPES = /** @type {const} */ ({
+  'image/png': 'png',
+  'image/jpeg': 'jpg',
+  'image/webp': 'webp',
+})
+
+/** @type {Map<number, keyof typeof MAGIC_BYTES>} */
+const magicByteMap = new Map()
+for (const [ext, bytes] of Object.entries(MAGIC_BYTES)) {
+  magicByteMap.set(
+    bytes[0],
+    // @ts-ignore
+    ext,
+  )
+}
+/**
+ * For a given buffer, determine the tile format based on the magic bytes.
+ * Will throw for unknown file types.
+ * Smaller and faster version of magic-bytes.js due to the limited use case.
+ *
+ * @param {Buffer | Uint8Array} buf
+ * @returns {import("../writer.js").TileFormat}
+ */
+export function getTileFormatFromBuffer(buf) {
+  const ext = magicByteMap.get(buf[0])
+  if (!ext) {
+    throw new Error('Unknown file type')
+  }
+  const sig = MAGIC_BYTES[ext]
+  for (let i = 1; i < sig.length; i++) {
+    if (typeof sig[i] !== 'undefined' && sig[i] !== buf[i]) {
+      throw new Error('Unknown file type')
+    }
+  }
+  if (ext === 'gz') {
+    // Gzipped tiles are always MVT
+    return 'mvt'
+  }
+  return ext
+}
+
+/**
+ * Determine the tile format from either a readable stream, buffer or Uint8Array
+ * from the magic bytes at the start of the file. Used if data is served without
+ * a content-type header.
+ *
+ * @param {import('stream').Readable} tileData
+ * @returns {Promise<[import("../writer.js").TileFormat, import('stream').Readable]>}
+ */
+export async function getTileFormatFromStream(tileData) {
+  // NB: The buffer-peek-stream library uses the peeked bytes as the high
+  // water mark for the transform stream, so we set this to the default high
+  // water mark of 16KB
+  const [peekedData, outputStream] = await peek(tileData, 16 * 1024)
+  tileData = outputStream
+  const format = getTileFormatFromBuffer(peekedData)
+  return [format, outputStream]
+}
+
+/**
+ * Get the tile format from a MIME type. Throws for unsupported types.
+ *
+ * @param {string} mimeType
+ * @returns {import("../writer.js").TileFormat}
+ */
+export function getFormatFromMimeType(mimeType) {
+  if (mimeType.startsWith('application/')) return 'mvt'
+  if (hasOwn(MIME_TYPES, mimeType)) return MIME_TYPES[mimeType]
+  throw new Error('Unsupported MIME type ' + mimeType)
+}

package/lib/utils/geo.js

@@ -0,0 +1,87 @@
+// Adapted from https://github.com/mapbox/tilebelt
+
+const r2d = 180 / Math.PI
+
+/** Spherical Mercator max bounds, rounded to 6 decimal places */
+export const MAX_BOUNDS = /** @type {BBox} */ ([
+  -180, -85.051129, 180, 85.051129,
+])
+
+/**
+ * @typedef {[number, number, number, number]} BBox
+ */
+
+/**
+ * Return the bounding box for the given tile.
+ *
+ * @param {{ x: number, y: number, z: number }} tile
+ * @returns {BBox} Bounding Box [w, s, e, n]
+ */
+export function tileToBBox({ x, y, z }) {
+  const e = tile2lon({ x: x + 1, z })
+  const w = tile2lon({ x, z })
+  const s = tile2lat({ y: y + 1, z })
+  const n = tile2lat({ y, z })
+  return [w, s, e, n]
+}
+
+/**
+ * @param {{ x: number, y: number, z: number }} tile
+ */
+export function getQuadkey({ x, y, z }) {
+  let quadkey = ''
+  let mask
+  for (let i = z; i > 0; i--) {
+    mask = 1 << (i - 1)
+    quadkey += (x & mask ? 1 : 0) + (y & mask ? 2 : 0)
+  }
+  return quadkey
+}
+
+/**
+ * From an array of tile URL templates, get the URL for the given tile.
+ *
+ * @param {string[]} urls
+ * @param {{ x: number, y: number, z: number, scheme?: 'xyz' | 'tms' }} opts
+ */
+export function getTileUrl(urls, { x, y, z, scheme = 'xyz' }) {
+  const bboxEspg3857 = tileToBBox({ x, y: Math.pow(2, z) - y - 1, z })
+  const quadkey = getQuadkey({ x, y, z })
+
+  return urls[(x + y) % urls.length]
+    .replace('{prefix}', (x % 16).toString(16) + (y % 16).toString(16))
+    .replace(/{z}/g, String(z))
+    .replace(/{x}/g, String(x))
+    .replace(/{y}/g, String(scheme === 'tms' ? Math.pow(2, z) - y - 1 : y))
+    .replace('{quadkey}', quadkey)
+    .replace('{bbox-epsg-3857}', bboxEspg3857.join(','))
+}
+
+/**
+ * Returns a bbox that is the smallest bounding box that contains all the input bboxes.
+ *
+ * @param {[BBox, ...BBox[]]} bboxes
+ * @returns {BBox} Bounding Box [w, s, e, n]
+ */
+export function unionBBox(bboxes) {
+  let [w, s, e, n] = bboxes[0]
+  for (let i = 1; i < bboxes.length; i++) {
+    const [w1, s1, e1, n1] = bboxes[i]
+    w = Math.min(w, w1)
+    s = Math.min(s, s1)
+    e = Math.max(e, e1)
+    n = Math.max(n, n1)
+  }
+  return [w, s, e, n]
+}
+
+/** @param {{ x: number, z: number }} opts */
+function tile2lon({ x, z }) {
+  return (x / Math.pow(2, z)) * 360 - 180
+}
+
+/** @param {{ y: number, z: number }} opts */
+function tile2lat({ y, z }) {
+  const n = Math.PI - (2 * Math.PI * y) / Math.pow(2, z)
+  return r2d * Math.atan(0.5 * (Math.exp(n) - Math.exp(-n)))
+}

package/lib/utils/mapbox.js

@@ -0,0 +1,155 @@
+'use strict'
+// from https://github.com/mapbox/mapbox-gl-js/blob/495a695/src/util/mapbox.js
+
+const API_URL = 'https://api.mapbox.com'
+const HELP = 'See https://www.mapbox.com/api-documentation/#access-tokens'
+
+/**
+ * @typedef {object} URLObject
+ * @property {string} protocol
+ * @property {string} authority
+ * @property {string} path
+ * @property {string[]} params
+ */
+
+/**
+ * @param {URLObject} urlObject
+ * @param {string} [accessToken]
+ */
+function makeAPIURL(urlObject, accessToken) {
+  const apiUrlObject = parseUrl(API_URL)
+  urlObject.protocol = apiUrlObject.protocol
+  urlObject.authority = apiUrlObject.authority
+
+  if (!accessToken) {
+    throw new Error(
+      `An API access token is required to use a Mapbox style. ${HELP}`,
+    )
+  }
+  if (accessToken[0] === 's') {
+    throw new Error(
+      `Use a public access token (pk.*) not a secret access token (sk.*). ${HELP}`,
+    )
+  }
+
+  urlObject.params.push(`access_token=${accessToken}`)
+  return formatUrl(urlObject)
+}
+
+/** @param {string} url */
+export function isMapboxURL(url) {
+  return url.indexOf('mapbox:') === 0
+}
+
+/**
+ * @param {string} url
+ * @param {string} [accessToken]
+ */
+export function normalizeStyleURL(url, accessToken) {
+  if (!isMapboxURL(url)) return url
+  if (!accessToken) throw new Error('Mapbox styles require an access token')
+  const urlObject = parseUrl(url)
+  urlObject.path = `/styles/v1${urlObject.path}`
+  return makeAPIURL(urlObject, accessToken)
+}
+
+/**
+ * @param {string} url
+ * @param {string} [accessToken]
+ */
+export function normalizeGlyphsURL(url, accessToken) {
+  if (!isMapboxURL(url)) return url
+  if (!accessToken) throw new Error('Mapbox styles require an access token')
+  const urlObject = parseUrl(url)
+  urlObject.path = `/fonts/v1${urlObject.path}`
+  return makeAPIURL(urlObject, accessToken)
+}
+
+/**
+ * @param {string} url
+ * @param {string} [accessToken]
+ */
+export function normalizeSourceURL(url, accessToken) {
+  if (!isMapboxURL(url)) return url
+  if (!accessToken) throw new Error('Mapbox styles require an access token')
+  const urlObject = parseUrl(url)
+  urlObject.path = `/v4/${urlObject.authority}.json`
+  // TileJSON requests need a secure flag appended to their URLs so
+  // that the server knows to send SSL-ified resource references.
+  urlObject.params.push('secure')
+  return makeAPIURL(urlObject, accessToken)
+}
+
+/**
+ * @param {string} url
+ * @param {'' | '@2x'} format
+ * @param {'.png' | '.json'} extension
+ * @param {string} [accessToken]
+ */
+export function normalizeSpriteURL(url, format, extension, accessToken) {
+  const urlObject = parseUrl(url)
+  if (!isMapboxURL(url)) {
+    urlObject.path += `${format}${extension}`
+    return formatUrl(urlObject)
+  }
+  urlObject.path = `/styles/v1${urlObject.path}/sprite${format}${extension}`
+  return makeAPIURL(urlObject, accessToken)
+}
+
+const imageExtensionRe = /(\.(png|jpg)\d*)(?=$)/
+
+/**
+ * @param {any} tileURL
+ * @param {string} sourceURL
+ * @param {256 | 512} [tileSize]
+ * @param {{ devicePixelRatio?: number; supportsWebp?: boolean; }} [opts]
+ */
+export function normalizeTileURL(
+  tileURL,
+  sourceURL,
+  tileSize,
+  { devicePixelRatio = 1, supportsWebp = false } = {},
+) {
+  if (!sourceURL || !isMapboxURL(sourceURL)) return tileURL
+
+  const urlObject = parseUrl(tileURL)
+
+  // The v4 mapbox tile API supports 512x512 image tiles only when @2x
+  // is appended to the tile URL. If `tileSize: 512` is specified for
+  // a Mapbox raster source force the @2x suffix even if a non hidpi device.
+  const suffix = devicePixelRatio >= 2 || tileSize === 512 ? '@2x' : ''
+  const extension = supportsWebp ? '.webp' : '$1'
+  urlObject.path = urlObject.path.replace(
+    imageExtensionRe,
+    `${suffix}${extension}`,
+  )
+
+  return formatUrl(urlObject)
+}
+
+const urlRe = /^(\w+):\/\/([^/?]*)(\/[^?]+)?\??(.+)?/
+
+/**
+ * @param {string} url
+ * @returns {URLObject}
+ */
+function parseUrl(url) {
+  const parts = url.match(urlRe)
+  if (!parts) {
+    throw new Error('Unable to parse URL object')
+  }
+  return {
+    protocol: parts[1],
+    authority: parts[2],
+    path: parts[3] || '/',
+    params: parts[4] ? parts[4].split('&') : [],
+  }
+}
+
+/**
+ * @param {URLObject} obj
+ */
+function formatUrl(obj) {
+  const params = obj.params.length ? `?${obj.params.join('&')}` : ''
+  return `${obj.protocol}://${obj.authority}${obj.path}${params}`
+}

package/lib/utils/misc.js

@@ -0,0 +1,26 @@
+/**
+ * Dumb and quick clone an object. Won't keep undefined properties. Types could
+ * be tighted so that return type excludes undefined properties, but not really
+ * needed.
+ *
+ * @template T
+ * @param {T} obj
+ * @returns {T}
+ */
+export function clone(obj) {
+  return JSON.parse(JSON.stringify(obj))
+}
+
+export function noop() {}
+
+/**
+ * Like `Object.hasOwn`, but refines the type of `key`.
+ *
+ * @template {Record<string, unknown>} T
+ * @param {T} obj
+ * @param {string} key
+ * @returns {key is (keyof T)}
+ */
+export function hasOwn(obj, key) {
+  return Object.hasOwn(obj, key)
+}