styled-map-package 1.0.0

package/.github/workflows/node.yml

@@ -0,0 +1,30 @@
+name: Node.js Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [18.x, 20.x, 22.x]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run tests
+        run: npm test

package/.github/workflows/release.yml

@@ -0,0 +1,47 @@
+name: release
+
+on:
+  workflow_dispatch:
+    inputs:
+      semver:
+        description: 'The semver to use'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          # - auto
+          - patch
+          - minor
+          - major
+          - prerelease
+          # - prepatch
+          # - preminor
+          # - premajor
+  pull_request:
+    types: [closed]
+    branches: [main]
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      issues: write
+      pull-requests: write
+    steps:
+      - name: Use Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - uses: actions/checkout@v4
+      - uses: nearform-actions/optic-release-automation-action@v4
+        with:
+          commit-message: 'Release {version}'
+          sync-semver-tags: true
+          access: 'public'
+          semver: ${{ github.event.inputs.semver }}
+          notify-linked-issues: false
+          # optional: set this secret in your repo config for publishing to NPM
+          npm-token: ${{ secrets.NPM_TOKEN }}
+          build-command: |
+            npm ci

package/.husky/pre-commit

@@ -0,0 +1 @@
+npx lint-staged

package/.nvmrc

@@ -0,0 +1 @@
+20

package/LICENSE.md

@@ -0,0 +1,7 @@
+Copyright (c) 2024 Awana Digital
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,28 @@
+# Styled Map Package
+
+A Styled Map Package (`.smp`) file is a Zip archive containing all the resources needed to serve a Maplibre vector styled map offline. This includes the style JSON, vector and raster tiles, glyphs (fonts), the sprite image, and the sprite metadata.
+
+## Installation
+
+Install globally to use the `smp` command.
+
+```sh
+npm install --global styled-map-package
+```
+
+## Usage
+
+Download an online map to a styled map package file, specifying the bounding box (west, south, east, north) and max zoom level.
+
+```sh
+smp download https://demotiles.maplibre.org/style.json \
+  --bbox '-180,-80,180,80' \
+  --zoom 5 \
+  --output demotiles.smp
+```
+
+Start a server and open in the default browser.
+
+```sh
+smp view demotiles.smp --open
+```

package/bin/smp-download.js

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+import { Command, InvalidArgumentError } from 'commander'
+import fs from 'fs'
+import { pipeline } from 'stream/promises'
+
+import download from '../lib/download.js'
+import { ttyReporter } from '../lib/reporters.js'
+
+const program = new Command()
+
+program
+  .description('Download a map style for offline usage')
+  .option('-o, --output [file]', 'output file (if omitted, writes to stdout)')
+  .requiredOption(
+    '-b, --bbox <west,south,east,north>',
+    'bounding box of area to download e.g. 11,47,12,47.5',
+    parseBbox,
+  )
+  .requiredOption(
+    '-z, --zoom <number>',
+    'max zoom level to download',
+    parseZoom,
+  )
+  .option(
+    '-t, --token <token>',
+    'Mapbox access token (necessary for Mapbox styles)',
+  )
+  .argument('<styleUrl>', 'URL to style to download', parseUrl)
+  .action(async (styleUrl, { bbox, zoom, output, token }) => {
+    const reporter = ttyReporter()
+    const readStream = download({
+      bbox,
+      maxzoom: zoom,
+      styleUrl,
+      onprogress: (p) => reporter.write(p),
+      accessToken: token,
+    })
+    const outputStream = output ? fs.createWriteStream(output) : process.stdout
+    await pipeline(readStream, outputStream)
+  })
+
+program.parseAsync(process.argv)
+
+/** @param {string} z */
+function parseZoom(z) {
+  const zoom = parseInt(z)
+  if (isNaN(zoom) || zoom < 0 || zoom > 22) {
+    throw new InvalidArgumentError(
+      'Zoom must be a whole number (integer) between 0 and 22.',
+    )
+  }
+  return zoom
+}
+
+/** @param {string} bbox */
+function parseBbox(bbox) {
+  const bounds = bbox.split(',').map((s) => parseFloat(s.trim()))
+  if (bounds.length !== 4) {
+    throw new InvalidArgumentError(
+      'Bounding box must have 4 values separated by commas.',
+    )
+  }
+  if (bounds.some(isNaN)) {
+    throw new InvalidArgumentError('Bounding box values must be numbers.')
+  }
+  return bounds
+}
+
+/** @param {string} url */
+function parseUrl(url) {
+  try {
+    return new URL(url).toString()
+  } catch (e) {
+    const message =
+      e !== null &&
+      typeof e === 'object' &&
+      'message' in e &&
+      typeof e.message === 'string'
+        ? e.message
+        : 'Invalid URL'
+    throw new InvalidArgumentError(message)
+  }
+}

package/bin/smp-view.js

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+import fastifyStatic from '@fastify/static'
+import { Command } from 'commander'
+import fastify from 'fastify'
+import openApp from 'open'
+
+import path from 'node:path'
+
+import smpServer from '../lib/server.js'
+
+const program = new Command()
+
+program
+  .description('Preview a styled map package in a web browser')
+  .option('-o, --open', 'open in the default web browser')
+  .option('-p, --port <number>', 'port to serve on', parseInt, 3000)
+  .argument('<file>', 'file to serve')
+  .action(async (filepath, { open, port }) => {
+    const address = await serve({ port, filepath })
+    console.log(`server listening on ${address}`)
+    if (open) {
+      await openApp(address)
+    }
+  })
+
+program.parseAsync(process.argv)
+
+/**
+ * Serve a styled map package on the given port (defaults to 3000). Use the
+ * fastify plugin in `./server.js` for more flexibility.
+ *
+ * @param {object} opts
+ * @param {number} [opts.port]
+ * @param {string} opts.filepath
+ * @returns
+ */
+function serve({ port = 3000, filepath }) {
+  const server = fastify()
+
+  server.register(fastifyStatic, {
+    root: new URL('../map-viewer', import.meta.url),
+    serve: false,
+  })
+  server.get('/', async (request, reply) => {
+    return reply.sendFile('index.html')
+  })
+
+  server.register(smpServer, {
+    filepath: path.relative(process.cwd(), filepath),
+  })
+  return server.listen({ port })
+}

package/bin/smp.js

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+import { Command } from 'commander'
+
+const program = new Command()
+
+program
+  .name('smp')
+  .command('download', 'Download a map style to a styled map package file')
+  .command('view', 'Preview a styled map package in a web browser')
+
+program.parse(process.argv)

package/eslint.config.js

@@ -0,0 +1,17 @@
+import js from '@eslint/js'
+import globals from 'globals'
+
+/** @type {import('eslint').Linter.FlatConfig[]} */
+export default [
+  js.configs.recommended,
+  {
+    languageOptions: {
+      ecmaVersion: 'latest',
+      sourceType: 'module',
+      globals: {
+        ...globals.node,
+      },
+    },
+    rules: {},
+  },
+]

package/lib/download.js

@@ -0,0 +1,114 @@
+import { Transform } from 'readable-stream'
+import { pipeline } from 'stream/promises'
+
+import Writer from '../lib/writer.js'
+import StyleDownloader from './style-downloader.js'
+
+/**
+ * @typedef {object} DownloadProgress
+ * @property {import('./tile-downloader.js').TileDownloadStats & { done: boolean }} tiles
+ * @property {{ done: boolean }} style
+ * @property {{ downloaded: number, done: boolean }} sprites
+ * @property {import('./style-downloader.js').GlyphDownloadStats & { done: boolean }} glyphs
+ * @property {{ totalBytes: number, done: boolean }} output
+ * @property {number} elapsedMs
+ */
+
+/**
+ * Download a map style and its resources for a given bounding box and max zoom
+ * level. Returns a readable stream of a "styled map package", a zip file
+ * containing all the resources needed to serve the style offline.
+ *
+ * @param {object} opts
+ * @param {import("./utils/geo.js").BBox} opts.bbox Bounding box to download tiles for
+ * @param {number} opts.maxzoom Max zoom level to download tiles for
+ * @param {string} opts.styleUrl URL of the style to download
+ * @param { (progress: DownloadProgress) => void } [opts.onprogress] Optional callback for reporting progress
+ * @param {string} [opts.accessToken]
+ * @returns {import('./types.js').DownloadStream} Readable stream of the output styled map file
+ */
+export default function download({
+  bbox,
+  maxzoom,
+  styleUrl,
+  onprogress,
+  accessToken,
+}) {
+  const downloader = new StyleDownloader(styleUrl, {
+    concurrency: 24,
+    mapboxAccessToken: accessToken,
+  })
+
+  let start = Date.now()
+  /** @type {DownloadProgress} */
+  let progress = {
+    tiles: { downloaded: 0, totalBytes: 0, total: 0, skipped: 0, done: false },
+    style: { done: false },
+    sprites: { downloaded: 0, done: false },
+    glyphs: { downloaded: 0, total: 0, totalBytes: 0, done: false },
+    output: { totalBytes: 0, done: false },
+    elapsedMs: 0,
+  }
+
+  const sizeCounter = new Transform({
+    transform(chunk, encoding, cb) {
+      handleProgress({
+        output: {
+          totalBytes: progress.output.totalBytes + chunk.length,
+          done: false,
+        },
+      })
+      cb(null, chunk)
+    },
+    final(cb) {
+      handleProgress({ output: { ...progress.output, done: true } })
+      cb()
+    },
+  })
+
+  /** @param {Partial<DownloadProgress>} update */
+  function handleProgress(update) {
+    progress = { ...progress, ...update, elapsedMs: Date.now() - start }
+    onprogress?.(progress)
+  }
+
+  ;(async () => {
+    const style = await downloader.getStyle()
+    const writer = new Writer(style)
+    handleProgress({ style: { done: true } })
+    writer.outputStream.pipe(sizeCounter)
+    writer.on('error', (err) => sizeCounter.destroy(err))
+
+    try {
+      for await (const spriteInfo of downloader.getSprites()) {
+        await writer.addSprite(spriteInfo)
+        handleProgress({
+          sprites: { downloaded: progress.sprites.downloaded + 1, done: false },
+        })
+      }
+      handleProgress({ sprites: { ...progress.sprites, done: true } })
+
+      const tiles = downloader.getTiles({
+        bounds: bbox,
+        maxzoom,
+        onprogress: (tileStats) =>
+          handleProgress({ tiles: { ...tileStats, done: false } }),
+      })
+      await pipeline(tiles, writer.createTileWriteStream({ concurrency: 24 }))
+      handleProgress({ tiles: { ...progress.tiles, done: true } })
+
+      const glyphs = downloader.getGlyphs({
+        onprogress: (glyphStats) =>
+          handleProgress({ glyphs: { ...glyphStats, done: false } }),
+      })
+      await pipeline(glyphs, writer.createGlyphWriteStream())
+      handleProgress({ glyphs: { ...progress.glyphs, done: true } })
+
+      writer.finish()
+    } catch (err) {
+      writer.outputStream.destroy(/** @type {Error} */ (err))
+    }
+  })()
+
+  return sizeCounter
+}

package/lib/index.js

@@ -0,0 +1,6 @@
+export { default as Reader } from './reader.js'
+export { default as Writer } from './writer.js'
+export { default as Server } from './server.js'
+export { default as StyleDownloader } from './style-downloader.js'
+export { downloadTiles } from './tile-downloader.js'
+export { default as download } from './download.js'

package/lib/reader.js

@@ -0,0 +1,150 @@
+import intoStream from 'into-stream'
+import { open } from 'yauzl-promise'
+
+import { json } from 'node:stream/consumers'
+
+import { validateStyle } from './utils/style.js'
+import {
+  getContentType,
+  getResourceType,
+  STYLE_FILE,
+  URI_BASE,
+} from './utils/templates.js'
+
+/**
+ * @typedef {object} Resource
+ * @property {string} resourceType
+ * @property {string} contentType
+ * @property {number} contentLength
+ * @property {import('stream').Readable} stream
+ * @property {'gzip'} [contentEncoding]
+ */
+
+/**
+ * A low-level reader for styled map packages. Returns resources in the package
+ * as readable streams, for serving over HTTP for example.
+ */
+export default class Reader {
+  /** @type {Promise<import('yauzl-promise').ZipFile>} */
+  #zipPromise
+  #entriesPromise
+  /** @type {undefined | Promise<void>} */
+  #closePromise
+
+  /**
+   * @param {string | import('yauzl-promise').ZipFile} filepathOrZip Path to styled map package (`.styledmap`) file, or an instance of yauzl ZipFile
+   */
+  constructor(filepathOrZip) {
+    const zipPromise = (this.#zipPromise =
+      typeof filepathOrZip === 'string'
+        ? open(filepathOrZip)
+        : Promise.resolve(filepathOrZip))
+    this.#entriesPromise = (async () => {
+      /** @type {Map<string, import('yauzl-promise').Entry>} */
+      const entries = new Map()
+      if (this.#closePromise) return entries
+      const zip = await zipPromise
+      if (this.#closePromise) return entries
+      for await (const entry of zip) {
+        if (this.#closePromise) return entries
+        entries.set(entry.filename, entry)
+      }
+      return entries
+    })()
+  }
+
+  /**
+   * Get the style JSON from the styled map package. The URLs in the style JSON
+   * will be transformed to use the provided base URL.
+   *
+   * @param {string | null} [baseUrl] Base URL where you plan to serve the resources in this styled map package, e.g. `http://localhost:3000/maps/styleA`
+   * @returns {Promise<import('./types.js').SMPStyle>}
+   */
+  async getStyle(baseUrl = null) {
+    const styleEntry = (await this.#entriesPromise).get(STYLE_FILE)
+    if (!styleEntry) throw new Error(`File not found: ${STYLE_FILE}`)
+    const stream = await styleEntry.openReadStream()
+    const style = await json(stream)
+    if (!validateStyle(style)) {
+      throw new AggregateError(validateStyle.errors, 'Invalid style')
+    }
+    if (typeof style.glyphs === 'string') {
+      style.glyphs = getUrl(style.glyphs, baseUrl)
+    }
+    if (typeof style.sprite === 'string') {
+      style.sprite = getUrl(style.sprite, baseUrl)
+    } else if (Array.isArray(style.sprite)) {
+      style.sprite = style.sprite.map(({ id, url }) => {
+        return { id, url: getUrl(url, baseUrl) }
+      })
+    }
+    for (const source of Object.values(style.sources)) {
+      if ('tiles' in source && source.tiles) {
+        source.tiles = source.tiles.map((tile) => getUrl(tile, baseUrl))
+      }
+    }
+    // Hard to get this type-safe without a validation function. Instead we
+    // trust the Writer and the tests for now.
+    return /** @type {import('./types.js').SMPStyle} */ (style)
+  }
+
+  /**
+   * Get a resource from the styled map package. The path should be relative to
+   * the root of the package.
+   *
+   * @param {string} path
+   * @returns {Promise<Resource>}
+   */
+  async getResource(path) {
+    if (path[0] === '/') path = path.slice(1)
+    if (path === STYLE_FILE) {
+      const styleJSON = JSON.stringify(await this.getStyle())
+      return {
+        contentType: 'application/json; charset=utf-8',
+        contentLength: Buffer.byteLength(styleJSON, 'utf8'),
+        resourceType: 'style',
+        stream: intoStream(styleJSON),
+      }
+    }
+    const entry = (await this.#entriesPromise).get(path)
+    if (!entry) throw new Error(`File not found: ${path}`)
+    const resourceType = getResourceType(path)
+    const contentType = getContentType(path)
+    const stream = await entry.openReadStream()
+    /** @type {Resource} */
+    const resource = {
+      resourceType,
+      contentType,
+      contentLength: entry.uncompressedSize,
+      stream,
+    }
+    if (path.endsWith('.gz')) {
+      resource.contentEncoding = 'gzip'
+    }
+    return resource
+  }
+
+  /**
+   * Close the styled map package file (should be called after reading the file to avoid memory leaks)
+   */
+  async close() {
+    if (this.#closePromise) return this.#closePromise
+    this.#closePromise = (async () => {
+      const zip = await this.#zipPromise
+      await zip.close()
+    })()
+    return this.#closePromise
+  }
+}
+
+/**
+ * @param {string} smpUri
+ * @param {string | null} baseUrl
+ */
+function getUrl(smpUri, baseUrl) {
+  if (!smpUri.startsWith(URI_BASE)) {
+    throw new Error(`Invalid SMP URI: ${smpUri}`)
+  }
+  if (typeof baseUrl !== 'string') return smpUri
+  return smpUri.replace(URI_BASE, baseUrl + '/')
+}

package/lib/reporters.js

@@ -0,0 +1,92 @@
+import chalk, { chalkStderr } from 'chalk'
+import logSymbols from 'log-symbols'
+import ora from 'ora'
+import prettyBytes from 'pretty-bytes'
+import prettyMilliseconds from 'pretty-ms'
+import { Writable } from 'readable-stream'
+
+chalk.level = chalkStderr.level
+
+const TASKS = /** @type {const} */ ([
+  'style',
+  'sprites',
+  'tiles',
+  'glyphs',
+  'output',
+])
+
+const TASK_LABEL = /** @type {const} */ ({
+  style: 'Downloading Map Style',
+  sprites: 'Downloading Sprites',
+  tiles: 'Downloading Tiles',
+  glyphs: 'Downloading Glyphs',
+  output: 'Writing Styled Map Package',
+})
+
+const TASK_SUFFIX =
+  /** @type {{ [K in (typeof TASKS)[number]]: (progress: import('./download.js').DownloadProgress[K]) => string }} */ ({
+    style: () => '',
+    sprites: ({ downloaded }) => `${downloaded}`,
+    tiles: ({ total, skipped, totalBytes, downloaded }) => {
+      const formattedTotal = total.toLocaleString()
+      const formattedCompleted = (downloaded + skipped)
+        .toLocaleString()
+        .padStart(formattedTotal.length)
+      return `${formattedCompleted}/${formattedTotal} (${prettyBytes(totalBytes)})`
+    },
+    glyphs: ({ total, downloaded, totalBytes }) =>
+      `${downloaded}/${total} (${prettyBytes(totalBytes)})`,
+    output: ({ totalBytes }) => `${prettyBytes(totalBytes)}`,
+  })
+
+/**
+ * A writable stream to reporting download progress to a TTY terminal. Write
+ * progress messages to this stream for a pretty-printed progress task-list in
+ * the terminal.
+ */
+export function ttyReporter() {
+  /** @type {import('./download.js').DownloadProgress | undefined} */
+  let stats
+  let current = 0
+  /** @type {import('ora').Ora} */
+  let spinner
+  return new Writable({
+    objectMode: true,
+    // @ts-ignore - missing type def
+    construct(cb) {
+      process.stderr.write('\n')
+      spinner = ora(TASK_LABEL[TASKS[current]]).start()
+      cb()
+    },
+    /** @param {ArrayLike<{ chunk: import('./download.js').DownloadProgress, encoding: string }>} chunks */
+    writev(chunks, cb) {
+      stats = chunks[chunks.length - 1].chunk
+      while (current < TASKS.length && stats[TASKS[current]].done) {
+        spinner.suffixText = chalk.dim(
+          TASK_SUFFIX[TASKS[current]](
+            // @ts-ignore - too complicated for TS
+            stats[TASKS[current]],
+          ),
+        )
+        spinner.succeed()
+        if (++current < TASKS.length) {
+          spinner = ora(TASK_LABEL[TASKS[current]]).start()
+        }
+      }
+      if (current < TASKS.length) {
+        spinner.suffixText = chalk.dim(
+          TASK_SUFFIX[TASKS[current]](
+            // @ts-ignore - too complicated for TS
+            stats[TASKS[current]],
+          ),
+        )
+      } else {
+        process.stderr.write(
+          `${chalk.green(logSymbols.success)} Completed in ${prettyMilliseconds(stats.elapsedMs)}\n`,
+        )
+      }
+
+      cb()
+    },
+  })
+}