tileserver-gl-light 5.5.0-pre.0 → 5.5.0-pre.11

package/src/main.js

@@ -12,7 +12,8 @@ import fs from 'node:fs';
 import fsp from 'node:fs/promises';
 import path from 'path';
 import { fileURLToPath } from 'url';
-import axios from 'axios';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
 import { server } from './server.js';
 import { isValidRemoteUrl } from './utils.js';
 import { openPMtiles, getPMtilesInfo } from './pmtiles_adapter.js';
@@ -81,7 +82,10 @@ program
   )
   .option(
     '-V, --verbose [level]',
-    'More verbose output (can specify level 1-3, default 1)',
+    'More verbose output (level 1-3)\n' +
+      '\t-V, --verbose, -V 1, or --verbose 1: Important operations\n' +
+      '\t-V 2 or --verbose 2: Detailed operations\n' +
+      '\t-V 3 or --verbose 3: All requests and debug info',
     (value) => {
       // If no value provided, return 1 (boolean true case)
       if (value === undefined || value === true) return 1;
@@ -91,6 +95,14 @@ program
       return isNaN(level) ? 1 : Math.min(Math.max(level, 1), 3);
     },
   )
+  .option(
+    '--fetch-timeout <ms>',
+    'External fetch timeout in milliseconds for renderer HTTP requests (default 15000)',
+    (value) => {
+      const v = parseInt(value, 10);
+      return isNaN(v) ? 15000 : v;
+    },
+  )
   .option('-s, --silent', 'Less verbose output')
   .option('-l|--log_file <file>', 'output log file (defaults to standard out)')
   .option(
@@ -118,6 +130,7 @@ const startServer = (configPath, config) => {
     silent: opts.silent,
     logFile: opts.log_file,
     logFormat: opts.log_format,
+    fetchTimeout: opts.fetchTimeout,
     publicUrl,
   });
 };
@@ -163,7 +176,6 @@ const startWithInputFile = async (inputFile) => {
     inputFile = path.resolve(process.cwd(), inputFile);
     inputFilePath = path.dirname(inputFile);
 
-    // eslint-disable-next-line security/detect-non-literal-fs-filename -- Validating local file from CLI argument
     const inputFileStats = await fsp.stat(inputFile);
     if (!inputFileStats.isFile() || inputFileStats.size === 0) {
       console.log(`ERROR: Not a valid input file: ${inputFile}`);
@@ -246,7 +258,7 @@ const startWithInputFile = async (inputFile) => {
       }
     }
 
-    if (opts.verbose) {
+    if (opts.verbose >= 1) {
       console.log(JSON.stringify(config, undefined, 2));
     } else {
       console.log('Run with --verbose to see the config file here.');
@@ -304,7 +316,7 @@ const startWithInputFile = async (inputFile) => {
       };
     }
 
-    if (opts.verbose) {
+    if (opts.verbose >= 1) {
       console.log(JSON.stringify(config, undefined, 2));
     } else {
       console.log('Run with --verbose to see the config file here.');
@@ -314,7 +326,6 @@ const startWithInputFile = async (inputFile) => {
   }
 };
 
-// eslint-disable-next-line security/detect-non-literal-fs-filename -- Config path from CLI argument is expected behavior
 fs.stat(path.resolve(opts.config), async (err, stats) => {
   if (err || !stats.isFile() || stats.size === 0) {
     let inputFile;
@@ -331,7 +342,6 @@ fs.stat(path.resolve(opts.config), async (err, stats) => {
       const files = await fsp.readdir(process.cwd());
       for (const filename of files) {
         if (filename.endsWith('.mbtiles') || filename.endsWith('.pmtiles')) {
-          // eslint-disable-next-line security/detect-non-literal-fs-filename -- Scanning current directory for tile files
           const inputFilesStats = await fsp.stat(filename);
           if (inputFilesStats.isFile() && inputFilesStats.size > 0) {
             inputFile = filename;
@@ -346,25 +356,26 @@ fs.stat(path.resolve(opts.config), async (err, stats) => {
         const url =
           'https://github.com/maptiler/tileserver-gl/releases/download/v1.3.0/zurich_switzerland.mbtiles';
         const filename = 'zurich_switzerland.mbtiles';
-        // eslint-disable-next-line security/detect-non-literal-fs-filename -- Writing demo file to known filename
+
         const writer = fs.createWriteStream(filename);
         console.log(`No input file found`);
         console.log(`[DEMO] Downloading sample data (${filename}) from ${url}`);
 
         try {
-          const response = await axios({
-            url,
-            method: 'GET',
-            responseType: 'stream',
-          });
-
-          response.data.pipe(writer);
-          writer.on('finish', () => startWithInputFile(filename));
-          writer.on('error', (err) =>
-            console.error(`Error writing file: ${err}`),
-          );
+          const response = await fetch(url);
+
+          if (!response.ok) {
+            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+          }
+
+          // Convert web ReadableStream to Node.js Readable stream and pipe to file
+          const nodeStream = Readable.fromWeb(response.body);
+          await pipeline(nodeStream, writer);
+
+          console.log('Download complete');
+          startWithInputFile(filename);
         } catch (error) {
-          console.error(`Error downloading file: ${error}`);
+          console.error(`Error downloading file: ${error.message || error}`);
         }
       }
     }

package/src/pmtiles_adapter.js

@@ -17,31 +17,45 @@ class S3Source {
    * @param {string} [s3Profile] - Optional AWS credential profile name from config.
    * @param {boolean} [configRequestPayer] - Optional flag from config for requester pays buckets.
    * @param {string} [configRegion] - Optional AWS region from config.
-   * @param {boolean} [verbose] - Whether to show verbose logging.
+   * @param {string} [s3UrlFormat] - Optional S3 URL format from config: 'aws' or 'custom'.
+   * @param {number} [verbose] - Verbosity level (1-3). 1=important, 2=detailed, 3=debug/all requests.
    */
   constructor(
     s3Url,
     s3Profile,
     configRequestPayer,
     configRegion,
+    s3UrlFormat,
     verbose = false,
   ) {
-    const parsed = this.parseS3Url(s3Url);
+    const parsed = this.parseS3Url(s3Url, s3UrlFormat);
     this.bucket = parsed.bucket;
     this.key = parsed.key;
     this.endpoint = parsed.endpoint;
     this.url = s3Url;
     this.verbose = verbose;
 
-    // Determine the final profile: Config takes precedence over URL
+    // Apply configuration precedence: Config > URL > Default
+    // Using || for strings (empty string = not set)
+    // Using ?? for booleans (false is valid value)
     const profile = s3Profile || parsed.profile;
-
-    // Determine requestPayer: Config takes precedence over URL
     this.requestPayer = configRequestPayer ?? parsed.requestPayer;
-
-    // Determine region: Config takes precedence over URL
     this.region = configRegion || parsed.region;
 
+    // Log precedence decisions for debugging
+    if (verbose >= 3) {
+      console.log(`S3 config precedence for ${s3Url}:`);
+      console.log(
+        `  Profile: ${s3Profile ? 'config' : parsed.profile ? 'url' : 'default'} = ${profile || 'none'}`,
+      );
+      console.log(
+        `  Region: ${configRegion ? 'config' : parsed.region !== (process.env.AWS_REGION || 'us-east-1') ? 'url' : 'env/default'} = ${this.region}`,
+      );
+      console.log(
+        `  RequestPayer: ${configRequestPayer !== undefined ? 'config' : parsed.requestPayer ? 'url' : 'default'} = ${this.requestPayer}`,
+      );
+    }
+
     // Create S3 client
     this.s3Client = this.createS3Client(
       parsed.endpoint,
@@ -54,60 +68,78 @@ class S3Source {
   /**
    * Parses various S3 URL formats into bucket, key, endpoint, region, and profile.
    * @param {string} url - The S3 URL to parse.
+   * @param {string} [s3UrlFormat] - Optional format override: 'aws' or 'custom'.
    * @returns {object} - An object containing bucket, key, endpoint, region, and profile.
    * @throws {Error} - Throws an error if the URL format is invalid.
    */
-  parseS3Url(url) {
-    // Initialize with defaults/environment
+  parseS3Url(url, s3UrlFormat) {
+    // Validate s3UrlFormat if provided
+    if (s3UrlFormat && s3UrlFormat !== 'aws' && s3UrlFormat !== 'custom') {
+      console.warn(
+        `Invalid s3UrlFormat: "${s3UrlFormat}". Must be "aws" or "custom". Using auto-detection.`,
+      );
+      s3UrlFormat = undefined;
+    }
+
     let region = process.env.AWS_REGION || 'us-east-1';
     let profile = null;
     let requestPayer = false;
 
-    const queryString = url.split('?')[1];
+    // Parse URL parameters
+    const [cleanUrl, queryString] = url.split('?');
     if (queryString) {
       const params = new URLSearchParams(queryString);
-      // Update profile if provided in url parameters
+      // URL parameters override defaults
       profile = params.get('profile') ?? profile;
-      // Update region if provided in url parameters
       region = params.get('region') ?? region;
-      // Update requestPayer if provided in url parameters
+      s3UrlFormat = s3UrlFormat ?? params.get('s3UrlFormat'); // Config overrides URL
+
       const payerVal = params.get('requestPayer');
       requestPayer = payerVal === 'true' || payerVal === '1';
     }
 
-    // Clean URL for format detection (remove trailing slashes)
-    const baseUrl = url.split('?')[0];
-    const cleanUrl = baseUrl.replace(/\/+$/, '');
+    // Helper to build result object
+    const buildResult = (endpoint, bucket, key) => ({
+      endpoint: endpoint ? `https://${endpoint}` : null,
+      bucket,
+      key,
+      region,
+      profile,
+      requestPayer,
+    });
 
-    // Format 1: s3://endpoint/bucket/key (S3-compatible storage)
-    // Example: s3://storage.example.com/mybucket/path/to/tiles.pmtile
-    const endpointMatch = cleanUrl.match(/^s3:\/\/([^\/]+)\/([^\/]+)\/(.+)$/);
-    if (endpointMatch) {
-      return {
-        endpoint: `https://${endpointMatch[1]}`,
-        bucket: endpointMatch[2],
-        key: endpointMatch[3],
-        region,
-        profile,
-        requestPayer,
-      };
-    }
+    // Define patterns based on format
+    const patterns = {
+      customWithDot: /^s3:\/\/([^/]*\.[^/]+)\/([^/]+)\/(.+)$/, // Auto-detect: requires dot
+      customForced: /^s3:\/\/([^/]+)\/([^/]+)\/(.+)$/, // Explicit: no dot required
+      aws: /^s3:\/\/([^/]+)\/(.+)$/,
+    };
 
-    // Format 2: s3://bucket/key (AWS S3 default)
-    // Example: s3://my-bucket/path/to/tiles.pmtiles
-    const awsMatch = cleanUrl.match(/^s3:\/\/([^\/]+)\/(.+)$/);
-    if (awsMatch) {
-      return {
-        endpoint: null, // Use default AWS endpoint
-        bucket: awsMatch[1],
-        key: awsMatch[2],
-        region,
-        profile,
-        requestPayer,
-      };
+    // Match based on s3UrlFormat or auto-detect
+    let match;
+
+    if (s3UrlFormat === 'custom') {
+      match = cleanUrl.match(patterns.customForced);
+      if (match) return buildResult(match[1], match[2], match[3]);
+    } else if (s3UrlFormat === 'aws') {
+      match = cleanUrl.match(patterns.aws);
+      if (match) return buildResult(null, match[1], match[2]);
+    } else {
+      // Auto-detection: try custom (with dot) first, then AWS
+      match = cleanUrl.match(patterns.customWithDot);
+      if (match) return buildResult(match[1], match[2], match[3]);
+
+      match = cleanUrl.match(patterns.aws);
+      if (match) return buildResult(null, match[1], match[2]);
     }
 
-    throw new Error(`Invalid S3 URL format: ${url}`);
+    throw new Error(
+      `Invalid S3 URL format: ${url}\n` +
+        `Expected formats:\n` +
+        `  AWS S3: s3://bucket-name/path/to/file.pmtiles\n` +
+        `  Custom endpoint: s3://endpoint.com/bucket/path/to/file.pmtiles\n` +
+        `Use s3UrlFormat parameter to override auto-detection if needed.`,
+    );
   }
 
   /**
@@ -115,7 +147,7 @@ class S3Source {
    * @param {string|null} endpoint - The custom endpoint URL, or null for default AWS S3.
    * @param {string} region - The AWS region.
    * @param {string} [profile] - Optional AWS credential profile name.
-   * @param {boolean} [verbose] - Whether to show verbose logging.
+   * @param {number} [verbose] - Verbosity level (1-3). 1=important, 2=detailed, 3=debug/all requests.
    * @returns {S3Client} - Configured S3Client instance.
    */
   createS3Client(endpoint, region, profile, verbose) {
@@ -276,13 +308,18 @@ async function readFileBytes(fd, buffer, offset) {
   });
 }
 
+// Cache for PMTiles objects to avoid creating multiple instances for the same URL
+const pmtilesCache = new Map();
+
 /**
  * Opens a PMTiles file from local filesystem, HTTP URL, or S3 URL.
+ * Uses caching to avoid creating multiple PMTiles instances for the same file.
  * @param {string} filePath - The path to the PMTiles file.
  * @param {string} [s3Profile] - Optional AWS credential profile name.
  * @param {boolean} [requestPayer] - Optional flag for requester pays buckets.
  * @param {string} [s3Region] - Optional AWS region.
- * @param {boolean} [verbose] - Whether to show verbose logging.
+ * @param {string} [s3UrlFormat] - Optional S3 URL format: 'aws' or 'custom'.
+ * @param {number} [verbose] - Verbosity level (1-3). 1=important, 2=detailed, 3=debug/all requests.
  * @returns {PMTiles} - A PMTiles instance.
  */
 export function openPMtiles(
@@ -290,8 +327,26 @@ export function openPMtiles(
   s3Profile,
   requestPayer,
   s3Region,
+  s3UrlFormat,
   verbose = 0,
 ) {
+  // Create a cache key that includes all parameters that affect the source
+  const cacheKey = JSON.stringify({
+    filePath,
+    s3Profile,
+    requestPayer,
+    s3Region,
+    s3UrlFormat,
+  });
+
+  // Check if we already have a PMTiles object for this configuration
+  if (pmtilesCache.has(cacheKey)) {
+    if (verbose >= 2) {
+      console.log(`Using cached PMTiles instance for: ${filePath}`);
+    }
+    return pmtilesCache.get(cacheKey);
+  }
+
   let pmtiles = undefined;
 
   if (isS3Url(filePath)) {
@@ -303,6 +358,7 @@ export function openPMtiles(
       s3Profile,
       requestPayer,
       s3Region,
+      s3UrlFormat,
       verbose,
     );
     pmtiles = new PMTiles(source);
@@ -316,12 +372,15 @@ export function openPMtiles(
     if (verbose >= 2) {
       console.log(`Opening PMTiles from local file: ${filePath}`);
     }
-    // eslint-disable-next-line security/detect-non-literal-fs-filename -- Opening local PMTiles file specified in config or CLI
+
     const fd = fs.openSync(filePath, 'r');
     const source = new PMTilesFileSource(fd);
     pmtiles = new PMTiles(source);
   }
 
+  // Cache the PMTiles object
+  pmtilesCache.set(cacheKey, pmtiles);
+
   return pmtiles;
 }
 

package/src/promises.js

@@ -8,7 +8,7 @@ export const existsP = async (path) => {
   try {
     await fsp.access(path); // Defaults to F_OK: indicating that the file is visible to the calling process
     return true;
-  } catch (err) {
+  } catch {
     return false;
   }
 };