tileserver-gl-light 5.5.0-pre.5 → 5.5.0-pre.7

package/src/main.js

@@ -82,7 +82,10 @@ program
   )
   .option(
     '-V, --verbose [level]',
-    'More verbose output (can specify level 1-3, default 1)',
+    'More verbose output (level 1-3)\n' +
+      '\t-V, --verbose, -V 1, or --verbose 1: Important operations\n' +
+      '\t-V 2 or --verbose 2: Detailed operations\n' +
+      '\t-V 3 or --verbose 3: All requests and debug info',
     (value) => {
       // If no value provided, return 1 (boolean true case)
       if (value === undefined || value === true) return 1;
@@ -92,6 +95,14 @@ program
       return isNaN(level) ? 1 : Math.min(Math.max(level, 1), 3);
     },
   )
+  .option(
+    '--fetch-timeout <ms>',
+    'External fetch timeout in milliseconds for renderer HTTP requests (default 15000)',
+    (value) => {
+      const v = parseInt(value, 10);
+      return isNaN(v) ? 15000 : v;
+    },
+  )
   .option('-s, --silent', 'Less verbose output')
   .option('-l|--log_file <file>', 'output log file (defaults to standard out)')
   .option(
@@ -119,6 +130,7 @@ const startServer = (configPath, config) => {
     silent: opts.silent,
     logFile: opts.log_file,
     logFormat: opts.log_format,
+    fetchTimeout: opts.fetchTimeout,
     publicUrl,
   });
 };
@@ -246,7 +258,7 @@ const startWithInputFile = async (inputFile) => {
       }
     }
 
-    if (opts.verbose) {
+    if (opts.verbose >= 1) {
       console.log(JSON.stringify(config, undefined, 2));
     } else {
       console.log('Run with --verbose to see the config file here.');
@@ -304,7 +316,7 @@ const startWithInputFile = async (inputFile) => {
       };
     }
 
-    if (opts.verbose) {
+    if (opts.verbose >= 1) {
       console.log(JSON.stringify(config, undefined, 2));
     } else {
       console.log('Run with --verbose to see the config file here.');

package/src/pmtiles_adapter.js

@@ -18,7 +18,7 @@ class S3Source {
    * @param {boolean} [configRequestPayer] - Optional flag from config for requester pays buckets.
    * @param {string} [configRegion] - Optional AWS region from config.
    * @param {string} [s3UrlFormat] - Optional S3 URL format from config: 'aws' or 'custom'.
-   * @param {boolean} [verbose] - Whether to show verbose logging.
+   * @param {number} [verbose] - Verbosity level (1-3). 1=important, 2=detailed, 3=debug/all requests.
    */
   constructor(
     s3Url,
@@ -147,7 +147,7 @@ class S3Source {
    * @param {string|null} endpoint - The custom endpoint URL, or null for default AWS S3.
    * @param {string} region - The AWS region.
    * @param {string} [profile] - Optional AWS credential profile name.
-   * @param {boolean} [verbose] - Whether to show verbose logging.
+   * @param {number} [verbose] - Verbosity level (1-3). 1=important, 2=detailed, 3=debug/all requests.
    * @returns {S3Client} - Configured S3Client instance.
    */
   createS3Client(endpoint, region, profile, verbose) {
@@ -308,14 +308,18 @@ async function readFileBytes(fd, buffer, offset) {
   });
 }
 
+// Cache for PMTiles objects to avoid creating multiple instances for the same URL
+const pmtilesCache = new Map();
+
 /**
  * Opens a PMTiles file from local filesystem, HTTP URL, or S3 URL.
+ * Uses caching to avoid creating multiple PMTiles instances for the same file.
  * @param {string} filePath - The path to the PMTiles file.
  * @param {string} [s3Profile] - Optional AWS credential profile name.
  * @param {boolean} [requestPayer] - Optional flag for requester pays buckets.
  * @param {string} [s3Region] - Optional AWS region.
  * @param {string} [s3UrlFormat] - Optional S3 URL format: 'aws' or 'custom'.
- * @param {boolean} [verbose] - Whether to show verbose logging.
+ * @param {number} [verbose] - Verbosity level (1-3). 1=important, 2=detailed, 3=debug/all requests.
  * @returns {PMTiles} - A PMTiles instance.
  */
 export function openPMtiles(
@@ -326,6 +330,23 @@ export function openPMtiles(
   s3UrlFormat,
   verbose = 0,
 ) {
+  // Create a cache key that includes all parameters that affect the source
+  const cacheKey = JSON.stringify({
+    filePath,
+    s3Profile,
+    requestPayer,
+    s3Region,
+    s3UrlFormat,
+  });
+
+  // Check if we already have a PMTiles object for this configuration
+  if (pmtilesCache.has(cacheKey)) {
+    if (verbose >= 2) {
+      console.log(`Using cached PMTiles instance for: ${filePath}`);
+    }
+    return pmtilesCache.get(cacheKey);
+  }
+
   let pmtiles = undefined;
 
   if (isS3Url(filePath)) {
@@ -357,6 +378,9 @@ export function openPMtiles(
     pmtiles = new PMTiles(source);
   }
 
+  // Cache the PMTiles object
+  pmtilesCache.set(cacheKey, pmtiles);
+
   return pmtiles;
 }
 

package/src/serve_data.js

@@ -58,7 +58,7 @@ export const serve_data = {
      * @returns {Promise<void>}
      */
     app.get('/:id/:z/:x/:y.:format', async (req, res) => {
-      if (verbose) {
+      if (verbose >= 1) {
         console.log(
           `Handling tile request for: /data/%s/%s/%s/%s.%s`,
           String(req.params.id).replace(/\n|\r/g, ''),
@@ -109,10 +109,10 @@ export const serve_data = {
         x,
         y,
       );
-      if (fetchTile == null && item.tileJSON.sparse) {
-        return res.status(410).send();
-      } else if (fetchTile == null) {
-        return res.status(204).send();
+      if (fetchTile == null) {
+        // sparse=true (default) -> 404 (allows overzoom)
+        // sparse=false -> 204 (empty tile, no overzoom)
+        return res.status(item.sparse ? 404 : 204).send();
       }
 
       let data = fetchTile.data;
@@ -180,7 +180,7 @@ export const serve_data = {
      */
     app.get('/:id/elevation/:z/:x/:y', async (req, res, next) => {
       try {
-        if (verbose) {
+        if (verbose >= 1) {
           console.log(
             `Handling elevation request for: /data/%s/elevation/%s/%s/%s`,
             String(req.params.id).replace(/\n|\r/g, ''),
@@ -261,7 +261,11 @@ export const serve_data = {
           xy[0],
           xy[1],
         );
-        if (fetchTile == null) return res.status(204).send();
+        if (fetchTile == null) {
+          // sparse=true (default) -> 404 (allows overzoom)
+          // sparse=false -> 204 (empty tile, no overzoom)
+          return res.status(item.sparse ? 404 : 204).send();
+        }
 
         let data = fetchTile.data;
         var param = {
@@ -294,7 +298,7 @@ export const serve_data = {
      * @returns {Promise<void>}
      */
     app.get('/:id.json', (req, res) => {
-      if (verbose) {
+      if (verbose >= 1) {
         console.log(
           `Handling tilejson request for: /data/%s.json`,
           String(req.params.id).replace(/\n|\r/g, ''),
@@ -331,7 +335,7 @@ export const serve_data = {
    * @param {string} id ID of the data source.
    * @param {object} programOpts - An object containing the program options
    * @param {string} programOpts.publicUrl Public URL for the data.
-   * @param {boolean} programOpts.verbose Whether verbose logging should be used.
+   * @param {number} programOpts.verbose Verbosity level (1-3). 1=important, 2=detailed, 3=debug/all requests.
    * @returns {Promise<void>}
    */
   add: async function (options, repo, params, id, programOpts) {
@@ -359,7 +363,7 @@ export const serve_data = {
       }
     }
 
-    if (verbose && verbose >= 1) {
+    if (verbose >= 1) {
       console.log(`[INFO] Loading data source '${id}' from: ${inputFile}`);
     }
 
@@ -381,7 +385,6 @@ export const serve_data = {
     tileJSON['format'] = 'pbf';
     tileJSON['encoding'] = params['encoding'];
     tileJSON['tileSize'] = params['tileSize'];
-    tileJSON['sparse'] = params['sparse'];
 
     if (inputType === 'pmtiles') {
       source = openPMtiles(
@@ -415,12 +418,20 @@ export const serve_data = {
       tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON);
     }
 
+    // Determine sparse: per-source overrides global, then format-based default
+    // sparse=true -> 404 (allows overzoom)
+    // sparse=false -> 204 (empty tile, no overzoom)
+    // Default: vector tiles (pbf) -> false, raster tiles -> true
+    const isVector = tileJSON.format === 'pbf';
+    const sparse = params.sparse ?? options.sparse ?? !isVector;
+
     // eslint-disable-next-line security/detect-object-injection -- id is from config file data source names
     repo[id] = {
       tileJSON,
       publicUrl,
       source,
       sourceType,
+      sparse,
     };
   },
 };

package/src/serve_font.js

@@ -36,7 +36,7 @@ export async function serve_font(options, allowedFonts, programOpts) {
       '',
     );
 
-    if (verbose) {
+    if (verbose >= 1) {
       console.log(
         `Handling font request for: /fonts/%s/%s.pbf`,
         sFontStack,
@@ -86,7 +86,7 @@ export async function serve_font(options, allowedFonts, programOpts) {
    * @returns {void}
    */
   app.get('/fonts.json', (req, res) => {
-    if (verbose) {
+    if (verbose >= 1) {
       console.log('Handling list font request for /fonts.json');
     }
     res.header('Content-type', 'application/json');

package/src/serve_rendered.js

@@ -1094,7 +1094,7 @@ export const serve_rendered = {
             (!p1 && p2 === 'static') || (p1 === 'static' && p2 === 'raw')
               ? 'static'
               : 'tile';
-          if (verbose) {
+          if (verbose >= 3) {
             console.log(
               `Handling rendered %s request for: /styles/%s%s/%s/%s/%s%s.%s`,
               requestType,
@@ -1154,7 +1154,7 @@ export const serve_rendered = {
         return res.sendStatus(404);
       }
       const tileSize = parseInt(req.params.tileSize, 10) || undefined;
-      if (verbose) {
+      if (verbose >= 3) {
         console.log(
           `Handling rendered tilejson request for: /styles/%s%s.json`,
           req.params.tileSize
@@ -1205,11 +1205,16 @@ export const serve_rendered = {
       renderersStatic: [],
       sources: {},
       sourceTypes: {},
+      sparseFlags: {},
     };
 
-    const { publicUrl, verbose } = programOpts;
+    const { publicUrl, verbose, fetchTimeout } = programOpts;
 
     const styleJSON = clone(style);
+
+    // Global sparse flag for HTTP/remote sources (from config options)
+    const globalSparse = options.sparse ?? true;
+
     /**
      * Creates a pool of renderers.
      * @param {number} ratio Pixel ratio
@@ -1231,7 +1236,7 @@ export const serve_rendered = {
           ratio,
           request: async (req, callback) => {
             const protocol = req.url.split(':')[0];
-            if (verbose && verbose >= 3) {
+            if (verbose >= 3) {
               console.log('Handling request:', req);
             }
             if (protocol === 'sprites') {
@@ -1287,22 +1292,18 @@ export const serve_rendered = {
                 x,
                 y,
               );
-              if (fetchTile == null && sourceInfo.sparse == true) {
-                if (verbose) {
-                  console.log(
-                    'fetchTile warning on %s, sparse response',
-                    req.url,
-                  );
+              if (fetchTile == null) {
+                if (verbose >= 2) {
+                  console.log('fetchTile null on %s', req.url);
                 }
-                callback();
-                return;
-              } else if (fetchTile == null) {
-                if (verbose) {
-                  console.log(
-                    'fetchTile error on %s, serving empty response',
-                    req.url,
-                  );
+                // eslint-disable-next-line security/detect-object-injection -- sourceId from internal style source names
+                const sparse = map.sparseFlags[sourceId] ?? true;
+                // sparse=true (default) -> return empty callback so MapLibre can overzoom
+                if (sparse) {
+                  callback();
+                  return;
                 }
+                // sparse=false -> 204 (empty tile, no overzoom) - create blank response
                 createEmptyResponse(
                   sourceInfo.format,
                   sourceInfo.color,
@@ -1341,40 +1342,58 @@ export const serve_rendered = {
 
               callback(null, response);
             } else if (protocol === 'http' || protocol === 'https') {
-              try {
-                // Add timeout to prevent hanging on unreachable hosts
-                const controller = new AbortController();
-                const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout
+              const controller = new AbortController();
+              const timeoutMs = (fetchTimeout && Number(fetchTimeout)) || 15000;
+              let timeoutId;
 
+              try {
+                timeoutId = setTimeout(() => controller.abort(), timeoutMs);
                 const response = await fetch(req.url, {
                   signal: controller.signal,
                 });
-
                 clearTimeout(timeoutId);
 
-                // Handle 410 Gone as sparse response
-                if (response.status === 410) {
-                  if (verbose) {
+                // HTTP 204 No Content means "empty tile" - generate a blank tile
+                if (response.status === 204) {
+                  const parts = url.parse(req.url);
+                  const extension = path.extname(parts.pathname).toLowerCase();
+                  // eslint-disable-next-line security/detect-object-injection -- extension is from path.extname, limited set
+                  const format = extensionToFormat[extension] || '';
+                  createEmptyResponse(format, '', callback);
+                  return;
+                }
+
+                if (!response.ok) {
+                  if (verbose >= 2) {
                     console.log(
-                      'fetchTile warning on %s, sparse response due to 410 Gone',
+                      'fetchTile HTTP %d on %s, %s',
+                      response.status,
                       req.url,
+                      globalSparse
+                        ? 'allowing overzoom'
+                        : 'creating empty tile',
                     );
                   }
-                  callback();
-                  return;
-                }
 
-                // Check for other non-ok responses
-                if (!response.ok) {
-                  throw new Error(
-                    `HTTP ${response.status}: ${response.statusText}`,
-                  );
+                  if (globalSparse) {
+                    // sparse=true -> allow overzoom
+                    callback();
+                    return;
+                  }
+
+                  // sparse=false (default) -> create empty tile
+                  const parts = url.parse(req.url);
+                  const extension = path.extname(parts.pathname).toLowerCase();
+                  // eslint-disable-next-line security/detect-object-injection -- extension is from path.extname, limited set
+                  const format = extensionToFormat[extension] || '';
+                  createEmptyResponse(format, '', callback);
+                  return;
                 }
 
                 const responseHeaders = response.headers;
                 const responseData = await response.arrayBuffer();
-
                 const parsedResponse = {};
+
                 if (responseHeaders.get('last-modified')) {
                   parsedResponse.modified = new Date(
                     responseHeaders.get('last-modified'),
@@ -1392,8 +1411,7 @@ export const serve_rendered = {
                 parsedResponse.data = Buffer.from(responseData);
                 callback(null, parsedResponse);
               } catch (error) {
-                // Log DNS failures more prominently as they often indicate config issues
-                // Native fetch wraps DNS errors in error.cause
+                // Log DNS failures
                 if (error.cause?.code === 'ENOTFOUND') {
                   console.error(
                     `DNS RESOLUTION FAILED for ${req.url}. ` +
@@ -1402,20 +1420,27 @@ export const serve_rendered = {
                   );
                 }
 
-                // Handle AbortController timeout
+                // Log timeout
                 if (error.name === 'AbortError') {
                   console.error(
                     `FETCH TIMEOUT for ${req.url}. ` +
-                      `The request took longer than 10 seconds to complete.`,
+                      `The request took longer than ${timeoutMs} ms to complete.`,
                   );
                 }
 
-                // For all other errors (e.g., network errors, 404, 500, etc.) return empty content.
+                // Log all other errors
                 console.error(
                   `Error fetching remote URL ${req.url}:`,
                   error.message || error,
                 );
 
+                if (globalSparse) {
+                  // sparse=true -> allow overzoom
+                  callback();
+                  return;
+                }
+
+                // sparse=false (default) -> create empty tile
                 const parts = url.parse(req.url);
                 const extension = path.extname(parts.pathname).toLowerCase();
                 // eslint-disable-next-line security/detect-object-injection -- extension is from path.extname, limited set
@@ -1497,7 +1522,7 @@ export const serve_rendered = {
 
         // Remove (flatten) 3D buildings
         if (layer.paint['fill-extrusion-height']) {
-          if (verbose) {
+          if (verbose >= 1) {
             console.warn(
               `Warning: Layer '${layerIdForWarning}' in style '${id}' has property 'fill-extrusion-height'. ` +
                 `3D extrusion may appear distorted or misleading when rendered as a static image due to camera angle limitations. ` +
@@ -1508,7 +1533,7 @@ export const serve_rendered = {
           layer.paint['fill-extrusion-height'] = 0;
         }
         if (layer.paint['fill-extrusion-base']) {
-          if (verbose) {
+          if (verbose >= 1) {
             console.warn(
               `Warning: Layer '${layerIdForWarning}' in style '${id}' has property 'fill-extrusion-base'. ` +
                 `3D extrusion may appear distorted or misleading when rendered as a static image due to camera angle limitations. ` +
@@ -1528,7 +1553,7 @@ export const serve_rendered = {
 
         for (const prop of hillshadePropertiesToRemove) {
           if (prop in layer.paint) {
-            if (verbose) {
+            if (verbose >= 1) {
               console.warn(
                 `Warning: Layer '${layerIdForWarning}' in style '${id}' has property '${prop}'. ` +
                   `This property is not supported by MapLibre Native. ` +
@@ -1543,7 +1568,7 @@ export const serve_rendered = {
 
         // --- Remove 'hillshade-shadow-color' if it is an array. It can only be a string in MapLibre Native ---
         if (Array.isArray(layer.paint['hillshade-shadow-color'])) {
-          if (verbose) {
+          if (verbose >= 1) {
             console.warn(
               `Warning: Layer '${layerIdForWarning}' in style '${id}' has property 'hillshade-shadow-color'. ` +
                 `An array value is not supported by MapLibre Native for this property (expected string/color). ` +
@@ -1589,7 +1614,6 @@ export const serve_rendered = {
 
     for (const name of Object.keys(styleJSON.sources)) {
       let sourceType;
-      let sparse;
       // eslint-disable-next-line security/detect-object-injection -- name is from style sources object keys
       let source = styleJSON.sources[name];
       let url = source.url;
@@ -1620,7 +1644,6 @@ export const serve_rendered = {
         if (dataInfo.inputFile) {
           inputFile = dataInfo.inputFile;
           sourceType = dataInfo.fileType;
-          sparse = dataInfo.sparse;
           s3Profile = dataInfo.s3Profile;
           requestPayer = dataInfo.requestPayer;
           s3Region = dataInfo.s3Region;
@@ -1664,7 +1687,6 @@ export const serve_rendered = {
           const type = source.type;
           Object.assign(source, metadata);
           source.type = type;
-          source.sparse = sparse;
           source.tiles = [
             // meta url which will be detected when requested
             `pmtiles://${name}/{z}/{x}/{y}.${metadata.format || 'pbf'}`,
@@ -1683,6 +1705,13 @@ export const serve_rendered = {
               tileJSON.attribution += source.attribution;
             }
           }
+
+          // Set sparse flag: user config overrides format-based default
+          // Vector tiles (pbf) default to false (204), raster tiles default to true (404)
+          const isVector = metadata.format === 'pbf';
+          // eslint-disable-next-line security/detect-object-injection -- name is from style sources object keys
+          map.sparseFlags[name] =
+            dataInfo.sparse ?? options.sparse ?? !isVector;
         } else {
           // MBTiles does not support remote URLs
 
@@ -1708,7 +1737,6 @@ export const serve_rendered = {
           const type = source.type;
           Object.assign(source, info);
           source.type = type;
-          source.sparse = sparse;
           source.tiles = [
             // meta url which will be detected when requested
             `mbtiles://${name}/{z}/{x}/{y}.${info.format || 'pbf'}`,
@@ -1731,6 +1759,13 @@ export const serve_rendered = {
               tileJSON.attribution += source.attribution;
             }
           }
+
+          // Set sparse flag: user config overrides format-based default
+          // Vector tiles (pbf) default to false (204), raster tiles default to true (404)
+          const isVector = info.format === 'pbf';
+          // eslint-disable-next-line security/detect-object-injection -- name is from style sources object keys
+          map.sparseFlags[name] =
+            dataInfo.sparse ?? options.sparse ?? !isVector;
         }
       }
     }

package/src/serve_style.js

@@ -35,7 +35,7 @@ export const serve_style = {
      */
     app.get('/:id/style.json', (req, res, next) => {
       const { id } = req.params;
-      if (verbose) {
+      if (verbose >= 1) {
         console.log(
           'Handling style request for: /styles/%s/style.json',
           String(id).replace(/\n|\r/g, ''),
@@ -95,7 +95,7 @@ export const serve_style = {
         const sanitizedFormat = format
           ? '.' + String(format).replace(/\n|\r/g, '')
           : '';
-        if (verbose) {
+        if (verbose >= 1) {
           console.log(
             `Handling sprite request for: /styles/%s/sprite/%s%s%s`,
             sanitizedId,
@@ -108,7 +108,7 @@ export const serve_style = {
         const item = repo[id];
         const validatedFormat = allowedSpriteFormats(format);
         if (!item || !validatedFormat) {
-          if (verbose)
+          if (verbose >= 1)
             console.error(
               `Sprite item or format not found for: /styles/%s/sprite/%s%s%s`,
               sanitizedId,
@@ -123,7 +123,7 @@ export const serve_style = {
         );
         const spriteScale = allowedSpriteScales(scale);
         if (!sprite || spriteScale === null) {
-          if (verbose)
+          if (verbose >= 1)
             console.error(
               `Bad Sprite ID or Scale for: /styles/%s/sprite/%s%s%s`,
               sanitizedId,
@@ -147,7 +147,7 @@ export const serve_style = {
 
         const sanitizedSpritePath = sprite.path.replace(/^(\.\.\/)+/, '');
         const filename = `${sanitizedSpritePath}${spriteScale}.${validatedFormat}`;
-        if (verbose) console.log(`Loading sprite from: %s`, filename);
+        if (verbose >= 1) console.log(`Loading sprite from: %s`, filename);
         try {
           const data = await readFile(filename);
 
@@ -156,7 +156,7 @@ export const serve_style = {
           } else if (validatedFormat === 'png') {
             res.header('Content-type', 'image/png');
           }
-          if (verbose)
+          if (verbose >= 1)
             console.log(
               `Responding with sprite data for /styles/%s/sprite/%s%s%s`,
               sanitizedId,
@@ -167,7 +167,7 @@ export const serve_style = {
           res.set({ 'Last-Modified': item.lastModified });
           return res.send(data);
         } catch (err) {
-          if (verbose) {
+          if (verbose >= 1) {
             console.error(
               'Sprite load error: %s, Error: %s',
               filename,
@@ -217,7 +217,28 @@ export const serve_style = {
     const styleFile = path.resolve(options.paths.styles, params.style);
     const styleJSON = clone(style);
 
-    const validationErrors = validateStyleMin(styleJSON);
+    // Sanitize style for validation: remove non-spec properties (e.g., 'sparse')
+    // so that validateStyleMin doesn't reject valid styles containing our custom flags.
+    const styleForValidation = clone(styleJSON);
+    if (styleForValidation.sources) {
+      for (const name of Object.keys(styleForValidation.sources)) {
+        if (
+          // eslint-disable-next-line security/detect-object-injection -- name is from Object.keys of styleForValidation.sources
+          styleForValidation.sources[name] &&
+          // eslint-disable-next-line security/detect-object-injection -- name is from Object.keys of styleForValidation.sources
+          'sparse' in styleForValidation.sources[name]
+        ) {
+          try {
+            // eslint-disable-next-line security/detect-object-injection -- name is from Object.keys of styleForValidation.sources
+            delete styleForValidation.sources[name].sparse;
+          } catch (_err) {
+            // ignore any deletion errors and continue validation
+          }
+        }
+      }
+    }
+
+    const validationErrors = validateStyleMin(styleForValidation);
     if (validationErrors.length > 0) {
       console.log(`The file "${params.style}" is not a valid style file:`);
       for (const err of validationErrors) {