tileblaster 0.4.8 → 1.0.0

package/UNLICENSE.md

@@ -0,0 +1,9 @@
+**This is free and unencumbered software released into the public domain.**
+
+Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means.
+
+In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to [unlicense.org](https://unlicense.org/)

package/bin/tileblaster.js

@@ -1,29 +1,79 @@
 #!/usr/bin/env node
-var argv = require('minimist')(process.argv.slice(2));
-var watch = require('node-watch');
-var configfile = require("path").resolve.apply(global, (!!argv._[0]) ? [ process.cwd(), argv._[0] ] : [ "../config.js" ]);
-var tb = require("../lib/tileblaster.js")(
-	(function(config){
-		["socket","tiles","queue","id","port"].forEach(function(n){
-			if (!!argv[n]) config[n] = argv[n];
+
+// run only if called directly
+if (require.main === module) {
+
+	const cluster = require("node:cluster");
+	const config = require("../lib/config");
+	const debug = require("../lib/debug");
+
+	if (cluster.isPrimary) { // main
+
+		// shutdown state
+		let shuttingdown = false;
+
+		// pool management
+		let pool = [];
+
+		// recursive fork helper
+		function fork(n){
+			worker = cluster.fork({ ...process.env, workerid: n });
+			debug.info("Started Worker #%d".bold.white, n);
+			worker.on('disconnect', function(){
+				debug.warn("Disconnect from Worker #%d".bold.white, n);
+				if (shuttingdown) { // remove worker from pool
+					pool[n] = null;
+					if (pool.filter(w=>!!w).length === 0) { // no more workers left, shutdowm
+						debug.info("All Workers terminated. Exiting.".bold.white);
+						process.exit(0);
+					}
+				} else { // fork new worker
+					pool[n] = fork(n);
+				}
+			});
+			return worker;
+		};
+
+		// initial fork for all worker threads
+		pool = Array(config.threads).fill().map(function(_,n){
+			return fork(n);
 		});
-		// watch changes in config file
-		watch(configfile, function(evt,f){
-			if (evt === "update") try {
-				delete require.cache[require.resolve(configfile)];
-				tb.reconfigure(require(configfile));
-			} catch (err) {
-				console.error("Unable to read changed config file", err);
-			}
+
+		const shutdown = function shutdown(){
+			shuttingdown = true;
+			debug.info("Shutdown initiated".white.bold);
+			pool.forEach(function(worker,n){
+				debug.info("Sending shutdown message to Worker #%d".white.bold, n);
+				worker.send("shutdown");
+			});
+			setTimeout(function(){
+				debug.warn("Exiting Non-Graceful".white.bold);
+				process.exit(1);
+			},2900);
+		};
+
+		process.on("SIGINT", shutdown);
+		process.on("SIGTERM", shutdown);
+
+		// watch config file for changes, reload workers on change
+		const watch = require("node-watch");
+		watch(config._file, {}).on("change", function(evt){
+			debug.info("Config file change detected, restarting Workers".white.bold);
+			pool.forEach(function(worker,n){
+				worker.send("shutdown");
+			});
 		});
-		return config;
-	})((function(){
-		try {
-			return require(configfile);
-		} catch (err) {
-			console.error("usage: tileblaster <config.js> [--socket tileblaster.sock] [--tiles /path/to/tiles] [--queue 100] [--id mytileblaster]");
-			process.exit(1);
-		}
-	})())
-).server().listen();
 
+		// create purge worker
+		const purge = require("../lib/purge");
+		setInterval(purge, 3600000).unref(); // regularly every  1h; FIXME: make configurable
+		setTimeout(purge, 300000).unref(); // run once after 5 minutes of uptime
+
+	} else { // worker
+
+		const tileblaster = require("../tileblaster");
+		const instance = tileblaster({ ...config, id: process.env.workerid });
+
+	}
+
+};

package/builtins/cache.js

@@ -0,0 +1,160 @@
+const fs = require("node:fs");
+const dur = require("dur");
+const path = require("node:path");
+const xattr = require("fs-xattr");
+
+const cache = {};
+
+// file cache
+module.exports = function({ opts, data, res }, next, skip){
+	const debug = this.lib.debug;
+	const config = this.config;
+
+	// cache opts
+	if (!cache.hasOwnProperty(data.map)) {
+
+		// store instance
+		cache.store = this.lib.store({ root: config.paths.data });
+
+		// merge opts of all uses of the cache builtin in a map config
+		opts = config.maps[data.map].filter(function(use){
+			return use.builtin === "cache";
+		}).reduce(function(opts, useopts){
+			return Object.entries(useopts).reduce(function(opts, [k,v]){
+				return opts[k]=v,opts;
+			}, opts);
+		},{});
+
+		cache[data.map] = {};
+
+		// skipto
+		cache[data.map].skipto = opts.skipto || false; // FIXME check valid label?
+
+		// true: expires instantly, false: expires never, number: seconds, string: parse
+		if (opts.hasOwnProperty("expires")) {
+			switch (typeof opts.expires) {
+				case "boolean":
+					cache[data.map].expires = opts.expires;
+				break;
+				case "number":
+					cache[data.map].expires = ((Number.isFinite(opts.expires)) ? Math.max(0,Math.round(opts.expires*1000)) : (opts.expires > 0)) || true;
+				break;
+				case "string":
+					cache[data.map].expires = dur(opts.expires, true);
+				break;
+				default:
+					cache[data.map].expires = true; // expires immediately
+				break;
+			}
+		} else {
+			cache[data.map].expires = false;
+		}
+
+	};
+	opts = cache[data.map];
+	const store = cache.store;
+
+	// if tile length is empty, retrieve. otherwise save
+	if (data.tiles.length === 0) {
+
+		// find extensions
+		let extensions = Object.entries(data.req.supports).reduce(function(extensions, [ext, supports]){
+			if (supports) extensions.push(ext);
+			return extensions;
+		},[]);
+
+		// get from storage
+		return store.find(data.req.path, extensions, function(err, tile){
+			if (err) return debug.error("Cache: Could not find %s: %s", data.req.path, err), next();
+			if (!tile) return next();
+
+			// check etag
+			if (data.req.etag && tile.attr.headers && data.req.etag === tile.attr.headers.etag) {
+				res.statusCode = 304;
+				res.end();
+				res.used = true;
+				return skip(); // skip rest of jobs
+			};
+
+			// check last-modified
+			if (data.req.last && tile.stats && data.req.last < tile.stats.mtimeMs) {
+				res.statusCode = 304;
+				res.end();
+				res.used = true;
+				return skip(); // skip rest of jobs
+			};
+
+			// if skipto is set, build tile data from cache and skip
+			if (opts.skipto) {
+
+				data.tile = tile.attr;
+				fs.readFile(tile.file, function(err, buf){
+					if (err) return debug.error("Cache: Could not read %s: %s", tile.file, err), next();
+
+					data.tile.buffer = buf;
+					data.tiles.push(data.tile);
+
+					skip(opts.skipto);
+				});
+
+			} else { // send tile to client
+
+				// if tile buffer is empty, return 204
+				if (tile.stats.size === 0) {
+					res.statusCode = 204;
+					res.used = true;
+					res.end();
+					return skip(); // skip rest of jobs
+				}
+
+				// set status and content-type
+				res.statusCode = tile.attr.status;
+
+				// set headers
+				Object.entries({
+					...tile.attr.headers, // tile-specific
+					"content-type": tile.attr.mimetype,
+					"content-length": tile.stats.size,
+				}).forEach(function([ k, v ]){
+					res.setHeader(k, v);
+				});
+
+				// send as stream
+				fs.createReadStream(tile.file).pipe(res);
+				res.used = true;
+
+				skip();
+
+			}
+
+		});
+
+	} else {
+
+		// set expires on all tiles
+		if (opts.expires !== false) {
+			let expires = Date.now()+opts.expires;
+			data.tile.expires = expires;
+			data.tiles.forEach(function(tile){
+				tile.expires = expires;
+			});
+		}
+
+		// no need to wait for files to get stored
+		next();
+
+		// skip storing if tiles expire immediately
+		if (opts.expires === true) return debug.warn("Cache: Skipping due to immediate expiration in map %s", data.map.magenta);
+
+		// store all tiles if not stored
+		Promise.allSettled(data.tiles.map(function(tile){
+			return new Promise(function(reject, resolve){
+				store.put(tile, function(err){
+					if (err) return debug.warn("Cache: Error storing %s: %s", tile.path.magenta, err), reject(err);
+					return debug.info("Cache: Stored %s", tile.path.magenta), resolve();
+				});
+			});
+		}));
+
+	};
+};

package/builtins/check.js

@@ -0,0 +1,115 @@
+// check data.params against constraints
+
+const cache = {};
+
+module.exports = function({ req, res, opts, data }, next){
+
+	// fill cache for map
+	if (!cache.hasOwnProperty(data.map)) {
+
+		cache[data.map] = {};
+
+		// assume reasonable default if no zoom level was specified
+		cache[data.map].zoom = (!opts.zoom || opts.zoom.length === 0) ? [ 0, 24 ] : opts.zoom;
+
+		// find min and max, clamp to reasonable levels
+		cache[data.map].minZoom = Math.max(0, Math.min(...cache[data.map].zoom));
+		cache[data.map].maxZoom = Math.min(24, Math.max(...cache[data.map].zoom));
+
+		// expand to array of zoom levels
+		cache[data.map].zoomLevels = Array(cache[data.map].maxZoom-cache[data.map].minZoom+1).fill().map(function(v,i){
+			return i+cache[data.map].minZoom;
+		});
+
+		if (opts.bbox && Array.isArray(opts.bbox) && opts.bbox.length === 4) {
+
+			// clamp bbox to planet and sort latitude
+			// longitude might be inverted when bounds span the antimeridian
+			cache[data.map].bbox = [
+				Math.max(-180, opts.bbox[0]),
+				Math.max( -90, Math.min(opts.bbox[1], opts.bbox[3])),
+				Math.min( 180, opts.bbox[2]),
+				Math.min(  90, Math.max(opts.bbox[1], opts.bbox[3])),
+			];
+
+			// precalculate tile index bounds per zoom level, clamp to planet
+			// hint: tile indexes increment north → south
+			// hint: https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames
+
+			// clamp tile index to max extent of zoom level
+			function clamp(v,z) {
+				return Math.max(0,Math.min(Math.pow(2,z)-1,v));
+			};
+
+			// tile x of zoom level from longitude
+			function lon(x,z) {
+				return clamp(Math.floor((x+180)/360*Math.pow(2,z)),z);
+			};
+
+			// tile y of zoom level from latitude
+			function lat(y,z){
+				return clamp(Math.floor((1-Math.log(Math.tan(y*Math.PI/180)+1/Math.cos(y*Math.PI/180))/Math.PI)/2*Math.pow(2,z)),z);
+			};
+
+			cache[data.map].bounds = cache[data.map].zoomLevels.reduce(function(b,z){
+				return b[z] = [
+					lon(cache[data.map].bbox[0],z), // west
+					lat(cache[data.map].bbox[3],z), // north
+					lon(cache[data.map].bbox[2],z), // east
+					lat(cache[data.map].bbox[1],z), // south
+				],b;
+			},[]);
+
+		} else {
+
+			cache[data.map].bbox = false;
+			cache[data.map].bounds = false;
+
+		}
+
+		// allowed extensions
+		cache[data.map].extensions = ((opts.extensions) ? ((Array.isArray(opts.extensions)) ? opts.extensions : ((typeof opts.extensions === "string") ? [ opts.extensions ] : [])) : []);
+
+		// density marker
+		if (opts.hasOwnProperty("density")) {
+			cache[data.map].density = (Array.isArray(opts.density) ? opts.density : [ opts.density ]).map(function(density){
+				return (!density) ? null : density;
+			});
+
+			if (cache[data.map].density.length === 0) cache[data.map].density = false;
+
+		} else {
+			cache[data.map].density = false;
+		}
+
+	};
+	opts = cache[data.map];
+
+	// check for NaNs
+	if (isNaN(data.params.z) || isNaN(data.params.x) || isNaN(data.params.y)) return next(new Error("illegal zxy."));
+
+	// check zoom
+	if (data.params.z < opts.minZoom || data.params.z > opts.maxZoom) return next(new Error("illegal zoom."));
+
+	// check bounds
+	if (opts.bounds) {
+		if (opts.bounds[data.params.z][0] < opts.bounds[data.params.z][2]) { // check for bounds spanning antimeridian
+			// bounds don't span antimeridian
+			if (data.params.x < opts.bounds[data.params.z][0] || data.params.x > opts.bounds[data.params.z][2]) return next(new Error("x is out of bounds."));
+		} else {
+			// bounds span antimeridian
+			if (data.params.x > opts.bounds[data.params.z][0] && data.params.x < opts.bounds[data.params.z][2]) return next(new Error("x is out of bounds, bounds span antimeridian"));
+		}
+		if (data.params.y < opts.bounds[data.params.z][1] || data.params.y > opts.bounds[data.params.z][3]) return next(new Error("y is out of bounds."));
+	}
+
+	// check extension
+	if (opts.extensions.length > 0 && !opts.extensions.includes(data.params.e) && !opts.extensions.includes(data.params.f)) return next(new Error("illegal extension."));
+
+	// check density
+	if (opts.density && !opts.density.includes(data.params.d) && !opts.density.includes(data.params.f)) return next(new Error("illegal density marker."));
+
+	// all passed
+	next();
+
+};

package/builtins/compress.js

@@ -0,0 +1,91 @@
+const zlib = require("node:zlib");
+const promisify = require("node:util").promisify;
+
+const brotli = promisify(zlib.brotliCompress);
+const gzip = promisify(zlib.gzip);
+
+const cache = {};
+
+// compress tiles
+module.exports = function({ req, res, opts, data }, next){
+	const debug = this.lib.debug;
+
+	if (!cache.hasOwnProperty(data.map)) cache[data.map] = {
+		brotli: (!opts.brotli) ? false :
+			(opts.brotli === true) ? { level: 9 } : // reasonable
+			(typeof opts.brotli === "number") ? { level: opts.brotli } :
+			(typeof opts.brotli === "object") ? opts.brotli : {},
+		gzip: (!opts.gzip) ? false :
+			(opts.gzip === true) ? { level: 9 } : // reasonable
+			(typeof opts.gzip === "number") ? { level: opts.gzip } :
+			(typeof opts.gzip === "object") ? opts.gzip : {},
+	};
+	opts = cache[data.map];
+
+	// pass through if nothing to do, but complain
+	if (!opts.brotli && !opts.gzip) {
+		debug.warn("No compression enabled for map '%s'", data.map);
+		return next();
+	};
+
+	// ensure unique tiles FIXME
+
+	// compress all uncompressed tiles
+	Promise.allSettled(data.tiles.filter(function(tile){
+		return (!tile.compression && tile.buffer.length > 0) // filter already compressed or  empty tiles
+	}).reduce(function(promises, tile){
+
+		// FIXME: roll this up:
+		if (opts.brotli) promises.push(new Promise(function(resolve, reject) {
+			brotli(tile.buffer, opts.brotli).then(function(compressed){
+				if (compressed.length > tile.buffer.length) {
+					debug.warn("Discarding useless Brotli compression for %s: +%db", data.path.magenta, compressed.length-tile.buffer.length);
+					return resolve();
+				};
+				data.tiles.push({
+					...tile,
+					buffer: compressed,
+					compression: "br",
+					headers: { ...(tile.headers||{}), "content-encoding": "br" },
+					params: { ...(tile.params||{}), c: ".br" },
+				});
+				resolve();
+			}).catch(function(err){
+				debug.error("Brotli failed for %s: %s", data.path.magenta, err);
+				reject(err);
+			});
+		}));
+
+		if (opts.gzip) promises.push(new Promise(function(resolve, reject) {
+			gzip(tile.buffer, opts.gzip).then(function(compressed){
+				if (compressed.length > tile.buffer.length) {
+					debug.warn("Discarding useless Gzip compression for %s: +%db", data.path.magenta, compressed.length-tile.buffer.length);
+					return resolve();
+				}
+				data.tiles.push({
+					...tile,
+					buffer: compressed,
+					compression: "gzip",
+					headers: { ...(tile.headers||{}), "content-encoding": "gzip" },
+					params: { ...(tile.params||{}), c: ".gz" },
+				});
+				resolve();
+			}).catch(function(err){
+				debug.error("Gzip failed for %s: %s", data.path.magenta, err);
+				reject(err);
+			});
+		}));
+
+		return promises;
+	}, [])).then(function(){
+
+		// set tile to best compressed tile client accepts FIXME find by buffer size?
+		const bestCompression = (opts.brotli && data.capabilities.br) ? "br" : (opts.gzip && data.capabilities.gz) ? "gzip" : null;
+		if (bestCompression) data.tile = data.tiles.find(function(tile){
+			return tile.compression === bestCompression && tile.mimetype === data.tile.mimetype && tile.filetype === data.tile.filetype;
+		}) || data.tile;
+
+		next();
+	});
+
+};

package/builtins/cors.js

@@ -0,0 +1,33 @@
+// send cors headers.
+
+module.exports = function({ req, res, opts, data }, next){
+
+	// only if origin header is set
+	if (!req.headers.hasOwnProperty("origin") || !req.headers.origin) return next();
+
+	// check origin in opts
+	if (!opts.hasOwnProperty("origins")) return next();
+
+	// ensure opts.origins is array of strings
+	if (!Array.isArray(opts.origins)) opts.origins = [ opts.origins ].filter(o=>typeof o === "string");
+
+	// check if origin is allowed cors
+	if (!opts.origins.includes("*") && opts.origins.includes(req.headers.origin)) return next();
+
+	// send common headers
+	res.setHeader("Access-Control-Allow-Origin", req.headers.origin||"*");
+	res.setHeader("Access-Control-Allow-Methods", "GET, OPTIONS");
+	res.setHeader("Access-Control-Allow-Headers", "DNT,If-None-Match,If-Modified-Since,Cache-Control,Content-Type,Range,Accept-Encoding");
+	res.setHeader("Access-Control-Expose-Headers", "Content-Length,Content-Range,Etag,Last-Modified,Content-Encoding");
+
+	// if method is OPTGIONS, set Access-Control-Max-Age and bail
+	if (req.method === "OPTIONS") {
+		res.setHeader("Access-Control-Max-Age", 1728000);
+		res.setHeader("Content-Type", "text/plain; charset=utf-8");
+		res.setHeader("Content-Length", 0);
+		res.statusCode = 204;
+		res.end();
+	};
+
+	next();
+};

package/builtins/deliver.js

@@ -0,0 +1,34 @@
+const stream = require("node:stream");
+
+// deliver tile
+module.exports = function({ res, opts, data }, next){
+
+	// do nothing if response stream has been used
+	if (res.used || res.headersSent || !res.writable || res.destroyed || res.finished || res.closed || res.piped) return next();
+
+	// if tile buffer is empty, return 204
+	if (data.tile.buffer.length === 0) {
+		res.statusCode = 204;
+		res.end();
+		return next();
+	}
+
+	// set status and content-type
+	res.statusCode = data.tile.status;
+
+	// set headers
+	Object.entries({
+		...data.tile.headers, // tile-specific
+		...opts.headers, // from opts
+		"content-type": data.tile.mimetype,
+		"content-length": data.tile.buffer.length,
+	}).forEach(function([ k, v ]){
+		res.setHeader(k, v);
+	});
+
+	// send as stream
+	stream.Readable.from(data.tile.buffer).pipe(res);
+	res.used = true;
+
+	next();
+};

package/builtins/dump.js

@@ -0,0 +1,19 @@
+const util = require("node:util");
+
+// a debuging module: dump all the information and end the request
+module.exports = function({ res, data }, next, skip){
+	const dump = util.inspect(data, false, 3);
+
+	console.error(dump);
+
+	// do nothing if response stream has been used
+	if (res.used || res.headersSent || !res.writable || res.destroyed || res.finished || res.closed || res.piped) return;
+
+	res.statusCode = 200;
+	res.setHeader("content-type", "text/plain");
+	res.end(dump);
+
+	res.used = true;
+	skip();
+
+};

package/builtins/edit.js

@@ -0,0 +1,30 @@
+const load = require("../lib/load");
+const vtt = load("vtt");
+
+const cache = {};
+
+// versatiles backend
+module.exports = function({ req, res, opts, data }, next){
+	const debug = this.lib.debug;
+
+	if (!vtt) return next(new Error("Dependency 'vtt' is missing."));
+
+	if (!opts.edit || typeof opts.edit !== "function") {
+		debug.warn("No edit function specified, skipping");
+		return next();
+	}
+
+	// apply to all tiles
+	data.tiles.filter(function(tile){
+		return tile.type === "pbf";
+	}).forEach(function(tile){
+		try {
+			tile.buffer = vtt.pack(opts.edit(vtt.unpack(tile.buffer)));
+		} catch (err) {
+			debug.error("Editing vector tile failed:", err);
+		}
+	});
+
+	next();
+
+};

package/builtins/mbtiles.js

@@ -0,0 +1,52 @@
+const load = require("../lib/load");
+const mbg = load("mbg");
+
+const cache = {};
+
+// mbtiles backend
+module.exports = function({ req, res, opts, data }, next){
+	const mime = this.lib.mime;
+	const debug = this.lib.debug;
+	const decompress = this.lib.decompress;
+
+	if (!mbg) return next(new Error("Dependency 'mbg' is missing."));
+
+	// cache mbg instance
+	if (!cache.hasOwnProperty(data.map)) cache[data.map] = new mbg(opts.file);
+	const mb = cache[data.map];
+
+	debug.info("Fetching %s/%s/%s", data.req.params.z, data.req.params.x, data.req.params.y);
+	mb.get(data.req.params.z, data.req.params.x, data.req.params.y, function(err, buf, info){
+		if (err) return next(err);
+
+		// decompress tile
+		decompress(buf, info.compression, function(err, buf){
+			if (err) return next(err);
+
+			// handle non-existant tiles
+			if (buf === null) return next(new Error("MBTiles: tile does not exist"));
+
+			const tile = {
+				buffer: buf,
+				type: mime.filetype(info.mimetype),
+				status: ((buf.length > 0) ? 200 : 204),
+				mimetype: info.mimetype,
+				// defaults
+				path: data.req.path,
+				headers: {},
+				compression: false,
+				language: null,
+				expires: true, // default policy
+			};
+
+			// add to tile stack, set primary tile
+			data.tiles.unshift(tile);
+			data.tile = tile;
+
+			next();
+
+		});
+
+	});
+
+};