tileserver-gl 5.5.0 → 5.6.0-pre.0

package/CHANGELOG.md

@@ -7,6 +7,10 @@
 ### 🐞 Bug fixes
 - _...Add new stuff here..._
 
+## 5.6.0-pre.0
+### ✨ Features and improvements
+- feat: Add ignore-missing-files cli option to avoid crashing at startup ([#1896](https://github.com/maptiler/tileserver-gl/pull/1896)) (by [andrewlaguna824](https://github.com/andrewlaguna824))
+
 ## 5.5.0
 - Add S3 support for PMTiles with multiple AWS credential profiles (https://github.com/maptiler/tileserver-gl/pull/1779) by @acalcutt
 - Create .aws directory passthrough folder in Dockerfile (https://github.com/maptiler/tileserver-gl/pull/1784) by @acalcutt

package/docs/usage.rst

@@ -20,6 +20,7 @@ Getting started
     -C|--no-cors              Disable Cross-origin resource sharing headers
     -u|--public_url <url>     Enable exposing the server on subpaths, not necessarily the root of the domain
     --fetch-timeout <ms>      Timeout in milliseconds for fetching remote tiles (default: 15000)
+    --ignore-missing-files    Do not exit when referenced data files or remote sources are missing at startup; log a warning and continue running (useful when styles reference optional or not-yet-available sources)
     -V, --verbose [level]     More verbose output (level 1-3)
                                 -V, --verbose, -V 1, or --verbose 1: Important operations
                                 -V 2 or --verbose 2: Detailed operations  

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tileserver-gl",
-  "version": "5.5.0",
+  "version": "5.6.0-pre.0",
   "description": "Map tile server for JSON GL styles - vector and server side generated raster tiles",
   "main": "src/main.js",
   "bin": {
@@ -37,7 +37,7 @@
     "docker": "docker build . && docker run --rm -i -p 8080:8080 $(docker build -q .)"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.962.0",
+    "@aws-sdk/client-s3": "^3.966.0",
     "@jsse/pbfont": "^0.3.0",
     "@mapbox/mapbox-gl-rtl-text": "0.3.0",
     "@mapbox/mbtiles": "0.12.1",
@@ -72,10 +72,10 @@
     "tileserver-gl-styles": "2.0.0"
   },
   "devDependencies": {
-    "@commitlint/cli": "^20.3.0",
+    "@commitlint/cli": "^20.3.1",
     "@commitlint/config-conventional": "^20.3.0",
-    "@typescript-eslint/eslint-plugin": "^8.51.0",
-    "@typescript-eslint/parser": "^8.51.0",
+    "@typescript-eslint/eslint-plugin": "^8.52.0",
+    "@typescript-eslint/parser": "^8.52.0",
     "chai": "6.2.2",
     "cross-env": "^10.1.0",
     "eslint": "^9.39.2",
@@ -89,7 +89,7 @@
     "pixelmatch": "^7.1.0",
     "prettier": "^3.7.4",
     "should": "^13.2.3",
-    "supertest": "^7.1.4",
+    "supertest": "^7.2.2",
     "yaml-lint": "^1.7.0"
   },
   "keywords": [

package/src/main.js

@@ -109,6 +109,10 @@ program
     '-f|--log_format <format>',
     'define the log format:  https://github.com/expressjs/morgan#morganformat-options',
   )
+  .option(
+    '--ignore-missing-files',
+    'Continue startup even if configured mbtiles/pmtiles files are missing',
+  )
   .version(packageJson.version, '-v, --version');
 program.parse(process.argv);
 const opts = program.opts();
@@ -132,6 +136,7 @@ const startServer = (configPath, config) => {
     logFormat: opts.log_format,
     fetchTimeout: opts.fetchTimeout,
     publicUrl,
+    ignoreMissingFiles: opts.ignoreMissingFiles,
   });
 };
 

package/src/serve_data.js

@@ -508,7 +508,7 @@ export const serve_data = {
    * @returns {Promise<void>}
    */
   add: async function (options, repo, params, id, programOpts) {
-    const { publicUrl, verbose } = programOpts;
+    const { publicUrl, verbose, ignoreMissingFiles } = programOpts;
     let inputFile;
     let inputType;
     if (params.pmtiles) {
@@ -542,8 +542,18 @@ export const serve_data = {
 
     // Only check file stats for local files, not remote URLs
     if (!isValidRemoteUrl(inputFile)) {
-      const inputFileStats = await fsp.stat(inputFile);
-      if (!inputFileStats.isFile() || inputFileStats.size === 0) {
+      try {
+        const inputFileStats = await fsp.stat(inputFile);
+        if (!inputFileStats.isFile() || inputFileStats.size === 0) {
+          throw Error(`Not valid input file: "${inputFile}"`);
+        }
+      } catch (err) {
+        if (ignoreMissingFiles) {
+          console.log(
+            `WARN: Data source '${id}' file not found: "${inputFile}" - skipping`,
+          );
+          return;
+        }
         throw Error(`Not valid input file: "${inputFile}"`);
       }
     }
@@ -555,24 +565,34 @@ export const serve_data = {
     tileJSON['encoding'] = params['encoding'];
     tileJSON['tileSize'] = params['tileSize'];
 
-    if (inputType === 'pmtiles') {
-      source = openPMtiles(
-        inputFile,
-        params.s3Profile,
-        params.requestPayer,
-        params.s3Region,
-        params.s3UrlFormat,
-        verbose,
-      );
-      sourceType = 'pmtiles';
-      const metadata = await getPMtilesInfo(source, inputFile);
-      Object.assign(tileJSON, metadata);
-    } else if (inputType === 'mbtiles') {
-      sourceType = 'mbtiles';
-      const mbw = await openMbTilesWrapper(inputFile);
-      const info = await mbw.getInfo();
-      source = mbw.getMbTiles();
-      Object.assign(tileJSON, info);
+    try {
+      if (inputType === 'pmtiles') {
+        source = openPMtiles(
+          inputFile,
+          params.s3Profile,
+          params.requestPayer,
+          params.s3Region,
+          params.s3UrlFormat,
+          verbose,
+        );
+        sourceType = 'pmtiles';
+        const metadata = await getPMtilesInfo(source, inputFile);
+        Object.assign(tileJSON, metadata);
+      } else if (inputType === 'mbtiles') {
+        sourceType = 'mbtiles';
+        const mbw = await openMbTilesWrapper(inputFile);
+        const info = await mbw.getInfo();
+        source = mbw.getMbTiles();
+        Object.assign(tileJSON, info);
+      }
+    } catch (err) {
+      if (ignoreMissingFiles) {
+        console.log(
+          `WARN: Unable to open data source '${id}' from "${inputFile}": ${err.message} - skipping (requests will return 404)`,
+        );
+        return;
+      }
+      throw err;
     }
 
     delete tileJSON['filesize'];

package/src/serve_style.js

@@ -213,7 +213,7 @@ export const serve_style = {
     reportTiles,
     reportFont,
   ) {
-    const { publicUrl } = programOpts;
+    const { publicUrl, ignoreMissingFiles } = programOpts;
     const styleFile = path.resolve(options.paths.styles, params.style);
     const styleJSON = clone(style);
 
@@ -247,6 +247,9 @@ export const serve_style = {
       return false;
     }
 
+    // Track missing sources
+    const missingSources = [];
+
     for (const name of Object.keys(styleJSON.sources)) {
       // eslint-disable-next-line security/detect-object-injection -- name is from Object.keys of style sources
       const source = styleJSON.sources[name];
@@ -270,7 +273,9 @@ export const serve_style = {
 
         const identifier = reportTiles(dataId, protocol);
         if (!identifier) {
-          return false;
+          // This datasource is missing or invalid in some way
+          missingSources.push(name);
+          continue;
         }
         source.url = `local://data/${identifier}.json`;
       }
@@ -286,6 +291,20 @@ export const serve_style = {
       }
     }
 
+    // Check if any sources are missing after processing all of them
+    if (missingSources.length > 0) {
+      if (ignoreMissingFiles) {
+        console.log(
+          `WARN: Style '${id}' references ${missingSources.length} missing data source(s): [${missingSources.join(', ')}] - not adding style`,
+        );
+      } else {
+        console.log(
+          `ERROR: Style '${id}' references missing data source(s): [${missingSources.join(', ')}]`,
+        );
+      }
+      return false;
+    }
+
     for (const obj of styleJSON.layers) {
       if (obj['type'] === 'symbol') {
         const fonts = (obj['layout'] || {})['text-font'];

package/src/server.js

@@ -241,7 +241,36 @@ async function start(opts) {
             }
           }
           if (dataItemId) {
-            // input files exists in the data config, return found id
+            // Data source exists in config, now validate file exists
+            // eslint-disable-next-line security/detect-object-injection -- dataItemId is validated above
+            const dataSource = data[dataItemId];
+            const fileType = dataSource.pmtiles ? 'pmtiles' : 'mbtiles';
+            const fileName = dataSource[fileType];
+
+            // Skip validation for remote URLs
+            if (fileName && !isValidRemoteUrl(fileName)) {
+              const filePath = path.resolve(options.paths[fileType], fileName);
+              try {
+                const stats = fs.statSync(filePath);
+                if (!stats.isFile() || stats.size === 0) {
+                  if (opts.ignoreMissingFiles) {
+                    // File doesn't exist or is empty - return undefined to skip
+                    return undefined;
+                  }
+                  // File missing but flag not set - let it fail later
+                  return dataItemId;
+                }
+              } catch (err) {
+                // File doesn't exist
+                if (opts.ignoreMissingFiles) {
+                  return undefined;
+                }
+                // File missing but flag not set - let it fail later
+                return dataItemId;
+              }
+            }
+
+            // File exists or is remote URL, return the id
             return dataItemId;
           } else {
             if (!allowMoreData) {
@@ -257,6 +286,21 @@ async function start(opts) {
               if (isValidRemoteUrl(styleSourceId)) {
                 id =
                   fnv1a(styleSourceId) + '_' + id.replace(/^.*\/(.*)$/, '$1');
+              } else {
+                try {
+                  const stats = fs.statSync(styleSourceId);
+                  if (!stats.isFile() || stats.size === 0) {
+                    if (opts.ignoreMissingFiles) {
+                      // File doesn't exist or is empty - return undefined to skip
+                      return undefined;
+                    }
+                  }
+                } catch (err) {
+                  // File doesn't exist
+                  if (opts.ignoreMissingFiles) {
+                    return undefined;
+                  }
+                }
               }
               // eslint-disable-next-line security/detect-object-injection -- id is being checked for existence before modification
               while (data[id]) id += '_'; //if the data source id already exists, add a "_" untill it doesn't