tileblaster 0.4.9 → 1.0.1

package/builtins/modernize.js

@@ -0,0 +1,57 @@
+// modernize
+module.exports = function modernize({ opts, data }, next){
+	const sharp = this.lib.sharp;
+	const debug = this.lib.debug;
+
+	// check if sharp is available
+	if (typeof sharp === "undefined") {
+		debug.warn("Modernize: Sharp dependency is missing.");
+		return next();
+	};
+
+	// check if tile should be optimized
+	if (!["png","jpeg","jpg","gif"].includes(data.tile.type)) {
+		debug.warn("Modernize: Unsupported file type: %s", data.tile.type);
+		return next();
+	};
+
+	Promise.allSettled(["avif","webp"].map(function(method){
+		return new Promise(function(resolve, reject) {
+			if (!opts[method]) return reject();
+
+			// copy primary tile
+			const tile = { ...data.tile };
+
+			// fixme: find suitable tile in tiles
+			sharp(tile.buffer).toFormat(method, opts[method]).toBuffer(function(err, buffer, info){
+				if (err) {
+					debug.error("Modernize: %s", err);
+					return reject();
+				};
+
+				// fix tile
+				tile.buffer = buffer;
+				tile.path = tile.path+"."+method;
+				tile.type = method;
+				tile.mimetype = "image/"+method;
+				tile.headers = { ...tile.headers };
+
+				// add to tile stack
+				data.tiles.unshift(tile);
+
+				// set primary if client supports
+				if (data.req.supports[method] && buffer.length < data.tile.buffer.length) {
+					debug.info("Modernized '%s': -%db", tile.path.magenta, data.tile.buffer.length-buffer.length);
+					data.tile = tile;
+				};
+
+				resolve();
+
+			});
+
+		});
+	})).then(function(){
+		next();
+	});
+
+};

package/builtins/noop.js

@@ -0,0 +1,5 @@
+// bare bones example builtin
+module.exports = function({ req, res, opts, data }, next){
+	console.log("I'm a plugin! my opts are: %s", JSON.stringify(opts));
+	next();
+};

package/builtins/optimize.js

@@ -0,0 +1,119 @@
+const worker = require("node:worker_threads");
+const stream = require("node:stream");
+
+if (worker.isMainThread) {
+
+	const cache = {};
+
+	// optimize
+	module.exports = function modernize({ opts, data }, next){
+		const debug = this.lib.debug;
+		const load = this.lib.load;
+
+		if (!cache.optipng) cache.optipng = load.exists("optipng-js");
+		if (!cache.mozjpeg) cache.mozjpeg = load.exists("js-mozjpeg");
+
+		if (!cache.hasOwnProperty(data.map)) cache[data.map] = {
+			png: (!opts.png) ? false : (opts.png === true) ? {} : opts.png,
+			jpeg: (!opts.jpeg) ? false : (opts.jpeg === true) ? {} : opts.jpeg,
+		};
+		opts = cache[data.map];
+
+		const jobs = [];
+
+		// find all pngs
+		if (opts.png && cache.optipng) /*[ data.tile, ...data.tiles ]*/data.tiles.filter(function(tile){
+			return (tile.type === "png" && !tile.compression && tile.buffer.length > 0) // uncompressed png only
+		}).forEach(function(tile){
+			jobs.push(new Promise(function(resolve,reject){
+
+				// create png worker
+				const w = new worker.Worker(__filename, {
+					workerData: { type: "png", opts: opts.png },
+					stdin: true, stdout: true,
+				});
+
+				// get result from worker
+				let result = [];
+				w.stdout.on("data", function(chunk){
+					result.push(chunk);
+				}).on("end", function(){
+					result = Buffer.concat(result);
+					if (result && result.length > 0 && result.length < tile.buffer.length) {
+						debug.info("Optimized PNG '%s': -%db", tile.path.magenta, tile.buffer.length-result.length);
+						tile.buffer = result;
+					}
+					resolve();
+				});
+
+				// stream buffer to worker
+				stream.Readable.from(tile.buffer).pipe(w.stdin);
+
+			}));
+		});
+
+		// find all jpegs
+		if (opts.jpeg && cache.mozjpeg) [ data.tile, ...data.tiles ].filter(function(tile){
+			return ((tile.type === "jpeg" || tile.type === "jpg") && !tile.compression && tile.buffer.length > 0) // uncompressed jpeg only
+		}).forEach(function(tile){
+			jobs.push(new Promise(function(resolve,reject){
+
+				// create jpeg worker
+				const w = new worker.Worker(__filename, {
+					workerData: { type: "jpeg", opts: opts.jpeg },
+					stdin: true, stdout: true,
+				});
+
+				// get result from worker
+				let result = [];
+				w.stdout.on("data", function(chunk){
+					result.push(chunk);
+				}).on("end", function(){
+					result = Buffer.concat(result);
+					if (result && result.length > 0 && result.length < tile.buffer.length) {
+						debug.info("Optimized JPEG '%s': -%db", tile.path.magenta, tile.buffer.length-result.length);
+						tile.buffer = result;
+					}
+					resolve();
+				});
+
+				// stream buffer to worker
+				stream.Readable.from(tile.buffer).pipe(w.stdin);
+
+			}));
+		});
+
+		// check if there are jobs
+		if (jobs.length === 0) return debug.warn("Nothing to optimize for '%s': -%db", data.tile.path.magenta), next();
+
+		// execute
+		Promise.allSettled(jobs).then(function(){
+			next();
+		});
+
+	};
+
+} else {
+	let buf = [];
+	process.stdin.on("data", function(chunk){
+		buf.push(chunk);
+	}).on("end", function() {
+		buf = Buffer.concat(buf);
+		if (buf.length > 0) switch (worker.workerData.type) {
+			case "png":
+				const optipng = require("optipng-js");
+				const outputpng = optipng(buf, { o: "3", i: "0", fix: true, quiet: true, ...worker.workerData.opts });
+				if (outputpng.data && outputpng.data.length > 0 && outputpng.data.length < buf.length) buf = Buffer.from(outputpng.data);
+			break;
+			case "jpg":
+			case "jpeg":
+				const jpegtran = require("js-mozjpeg").jpegtran;
+				const outputjpeg = jpegtran(buf, { optimize: true, maxmemory: "4m", copy: "none", ...(worker.workerData.opts||{}) });
+				if (outputjpeg.data && outputjpeg.data.length > 0 && outputjpeg.data.length < buf.length) buf = Buffer.from(outputjpeg.data);
+			break;
+		};
+
+		stream.Readable.from(buf).pipe(process.stdout);
+	});
+
+};

package/builtins/parse.js

@@ -0,0 +1,52 @@
+// parse client request
+
+module.exports = function({ req, res, opts, data }, next){
+
+	data.req = {};
+
+	data.req.path = req.path;
+
+	// cache things
+	data.req.etag = req.headers["if-none-match"] || null;
+	data.req.last = req.headers.hasOwnProperty("if-modified-since") ? Date.parse(req.headers["if-modified-since"]).valueOf() : null;
+
+	// examine some headers for capabilities
+	data.req.supports = {
+		webp: (req.headers.accept||"").includes("image/webp"),
+		avif: (req.headers.accept||"").includes("image/avif"),
+		br: (req.headers["accept-encoding"]||"").includes("br"),
+		gz: (req.headers["accept-encoding"]||"").includes("gzip"),
+	};
+
+	// languages
+	data.req.lang = ((!!req.headers["accept-language"]) ? req.headers["accept-language"].split(",").map(function(lang){
+		return lang.split(";").shift().trim();
+	}) : []).map(function(lang){
+		return lang.slice(0,2).toLowerCase();
+	}).filter(function(lang,i,languages){
+		return languages.indexOf(lang) === i;
+	});
+
+	// get params from steps
+	data.req.params = {
+		m: data.map,
+		x: parseInt(req.steps[2],10), // lon
+		y: parseInt(req.steps[3],10), // lat
+		z: parseInt(req.steps[1],10), // zoom
+		r: req.steps[3].includes("@") ? req.steps[3].slice(req.steps[3].indexOf("@"), req.steps[3].indexOf("x")+1) : "", // raw density marker ("@2x")
+		e: req.steps[3].includes(".") ? req.steps[3].slice(req.steps[3].indexOf(".", Math.max(0,req.steps[3].indexOf("x")))+1) : null, // extension
+	};
+
+	// density options
+	data.req.params.d = data.req.params.r ? parseFloat(data.req.params.r.slice(1,-1)) : 1;
+	data.req.params.w = Math.round(data.req.params.d * 256);
+
+	// patch in override parse function
+	if (opts.hasOwnProperty("parse") && typeof opts.parse === "function") return opts.parse(req, function(err, params){
+		if (err) return next(err);
+		data.req.params = { ...data.req.params, params };
+		next();
+	});
+
+	next();
+};

package/builtins/pmtiles.js

@@ -0,0 +1,80 @@
+const load = require("../lib/load");
+const pmtiles = load("pmtiles");
+
+// map pmtiles compression id to method
+const pmcompression = [ null, null, "gz", "br", "zstd" ];
+
+const cache = {};
+
+// pmtiles backend
+module.exports = function({ req, res, opts, data }, next){
+	const mime = this.lib.mime;
+	const debug = this.lib.debug;
+	const decompress = this.lib.decompress;
+
+	if (!pmtiles) return next(new Error("Dependency 'pmtiles' is missing."));
+
+	const run = function(){
+		const pm = cache[data.map];
+
+		debug.info("Fetching %s/%s/%s", data.req.params.z, data.req.params.x, data.req.params.y);
+		pm.tiles.getZxy(data.req.params.z, data.req.params.x, data.req.params.y).then(function(result){
+
+			// undefined result means tile does not exists
+			if (!result) return next(new Error("PMTiles: tile does not exist"));
+
+			// pm.tiles.decompress(Buffer.from(result.data), pm.header.tileCompression).then(function(buf){
+			// pmtiles spec allows gzip, brotli and zstd, decompress function only supports gzip. use own implementation?
+			decompress(Buffer.from(result.data), pmcompression[pm.header.tileCompression]).then(function(buf){
+
+				const tile = {
+					buffer: buf,
+					type: pm.type,
+					status: ((buf.length > 0) ? 200 : 204),
+					mimetype: mime.mimetype([pm.type]),
+					// defaults
+					path: data.req.path,
+					headers: {},
+					compression: false,
+					language: null,
+					expires: true, // default policy
+				};
+
+				// add to tile stack, set primary tile
+				data.tiles.unshift(tile);
+				data.tile = tile;
+
+				next();
+
+			}).catch(function(err){
+				return next(err);
+			});
+
+		}).catch(function(err){
+			return next(err);
+		});
+
+	};
+
+	if (cache.hasOwnProperty(data.map)) return run();
+
+	cache[data.map] = {};
+	cache[data.map].tiles = new pmtiles.PMTiles(opts.url);
+
+	cache[data.map].tiles.getHeader().then(function(header){
+
+		cache[data.map].header = header;
+
+		switch (header.tileType) {
+			case 0: cache[data.map].type = "bin"; break;
+			case 1: cache[data.map].type = "pbf"; break;
+			case 2: cache[data.map].type = "png"; break;
+			case 3: cache[data.map].type = "jpeg"; break;
+			case 4: cache[data.map].type = "webp"; break;
+		};
+
+		run();
+
+	});
+
+};

package/builtins/sharp.js

@@ -0,0 +1,46 @@
+
+// sharp image manipulation
+module.exports = function sharp({ opts, data }, next){
+	const sharp = this.lib.sharp;
+	const debug = this.lib.debug;
+
+	// check if sharp is available
+	if (typeof sharp === "undefined") {
+		debug.warn("sharp: `sharp` dependency is missing.");
+		return next();
+	};
+
+	// check if image can be edited
+	if (!["png","jpeg","jpg","gif","webp","avif","tif","tiff"].includes(data.tile.type)) {
+		debug.warn("sharp: Unsupported file type: %s", data.tile.type);
+		return next();
+	};
+
+	let img = sharp(data.tile.buffer);
+	let error = null;
+
+	Object.entries(opts).filter(function([method, param]){
+		return (typeof img[method] === "function");
+	}).forEach(function([method, param]){
+		if (error) return;
+		try {
+			img = img[method](param);
+		} catch (err) {
+			debug.error("sharp: %s failed", method, err);
+			img = null, error = err;
+		}
+	});
+
+	// end on previous error
+	if (error) return next(error);
+
+	// write buffer to tile
+	img.toBuffer().then(function(buf){
+		data.tile.buffer = buf;
+		next();
+	}).catch(function(err){
+		debug.error("sharp: creating image buffer failed", err);
+		return next(err);
+	});
+
+};

package/builtins/tileserver.js

@@ -0,0 +1,106 @@
+const cache = {};
+
+// tileserver backend
+module.exports = function({ req, res, opts, data }, next){
+	const mime = this.lib.mime;
+	const debug = this.lib.debug;
+	const strtpl = this.lib.strtpl;
+	const retrieve = this.lib.retrieve;
+
+	// cache opts
+	if (!cache.hasOwnProperty(data.map)) {
+		cache[data.map] = {};
+
+		cache[data.map].url = opts.url;
+
+		// success codes, convert to int, filter crap, default 200
+		cache[data.map].status = (opts.hasOwnProperty("status")) ? Array.isArray(opts.status) ? opts.status : [ opts.status ] : [ 200 ];
+		cache[data.map].status.map(function(status){ return parseInt(status,10); }).filter(function(status){ return !isNaN(status); });
+		if (cache[data.map].status.length === 0) cache[data.map].status.push(200);
+
+		// headers
+		cache[data.map].headers = opts.headers || {};
+
+		// mime types, make array
+		cache[data.map].mimetypes = (opts.hasOwnProperty("mimetypes")) ? Array.isArray(opts.mimetypes) ? opts.mimetypes : [ opts.mimetypes ] : false; // FIXME tolowercase
+
+		// tms
+		cache[data.map].tms = opts.tms === true;
+
+		// subdomains
+		cache[data.map].subdomains = (opts.hasOwnProperty("subdomains")) ? Array.isArray(opts.subdomains) ? opts.subdomains : [ opts.subdomains ] : false;
+
+		// headers
+		cache[data.map].headers = (opts.hasOwnProperty("headers")) ? opts.headers : {};
+
+	};
+	opts = cache[data.map];
+
+	// clone params for request
+	const params = { ...data.req.params };
+
+	// flip y for tms maps
+	if (opts.tms) params.y = Math.pow(2,params.z)-params.y-1;
+
+	// set subdomain if configured
+	if (opts.subdomains) params.s = opts.subdomains[ Date.now() % opts.subdomains.length ];
+
+	const tileurl = strtpl(opts.url, params);
+
+	// FIXME check fails cache
+
+	debug.info("Fetching %s", tileurl);
+
+	// request
+	retrieve({
+		url: tileurl,
+		headers: {
+			...opts.headers,
+		},
+		followRedirects: true,
+		compression: true,
+		timeout: 10000,
+	}).then(function(resp){
+
+		// FIXME cache fails
+
+		// check response status code
+		if (opts.status && !opts.status.includes(resp.statusCode)) return next(new Error("Source Tileserver responded with statusCode "+resp.statusCode)); // FIXME set response status?
+
+		// get respose media type
+		const contenttype = (resp.headers["content-type"]||"application/octet-stream").trim().toLowerCase();
+		const mediatype = contenttype.includes(";") ? contenttype.slice(0, contenttype.indexOf(";")) : contenttype;
+
+		// check response mime type
+		if (opts.mimetypes && !opts.mimetypes.includes(mediatype)) return next(new Error("Source Tileserver responded with disallowed mime-type "+mediatype));
+
+		// set tile
+		const tile = {};
+		tile.path = data.req.path;
+		tile.buffer = resp.body;
+		tile.mimetype = opts.mimetype || mediatype; // override via opts
+		tile.type = opts.filetype || mime.filetype(opts.mimetype || mediatype, data.req.params.e);
+
+		// http response
+		tile.status = (tile.buffer.length > 0) ? 200 : 204;
+
+		// defaults
+		tile.headers = {};
+		tile.compression = false; // http client always delivers uncompressed
+		tile.language = null;
+		tile.expires = true; // default policy
+
+		// keep around? FIXME
+		// data.tile.sourceHeaders = resp.headers;
+
+		// add to tile stack, set primary tile
+		data.tiles.unshift(tile);
+		data.tile = tile;
+
+		next();
+
+	}).catch(function(err){
+		next(err);
+	});
+
+};

package/builtins/versatiles.js

@@ -0,0 +1,68 @@
+const load = require("../lib/load");
+const versatiles = load("versatiles");
+
+const cache = {};
+
+// versatiles backend
+module.exports = function({ req, res, opts, data }, next){
+	const mime = this.lib.mime;
+	const debug = this.lib.debug;
+
+	if (!versatiles) return next(new Error("Dependency 'versatiles' is missing."));
+
+	// cache versatiles instance
+	if (!cache.hasOwnProperty(data.map)) {
+		cache[data.map] = new versatiles(opts.url, {
+			tms: (!!opts.tms),
+			headers: ((opts.hasOwnProperty("headers")) ? opts.headers : {}),
+		});
+	};
+	const vt = cache[data.map];
+
+	debug.info("Fetching %s/%s/%s", data.req.params.z, data.req.params.x, data.req.params.y);
+	vt.getTile(data.req.params.z, data.req.params.x, data.req.params.y, function(err, buf){
+		if (err) return next(err);
+
+		// if precompressed, keep in tile stack
+		/* TODO evaluate side effects
+		if (vt.header.tile_precompression) data.tiles.unshift({
+			buffer: buf,
+			type: vt.header.tile_format,
+			status: ((buf.length > 0) ? 200 : 204),
+			mimetype: vt.mimetypes[vt.header.tile_format],
+			// defaults
+			path: data.req.path,
+			headers: {},
+			compression: vt.header.tile_precompression,
+			language: null,
+			expires: true, // default policy
+		}); */
+
+		// decompress tile
+		vt.decompress(vt.header.tile_precompression, buf, function(err, buf){
+			if (err) return next(err);
+
+			const tile = {
+				buffer: buf,
+				type: vt.header.tile_format,
+				status: ((buf.length > 0) ? 200 : 204),
+				mimetype: vt.mimetypes[vt.header.tile_format],
+				// defaults
+				path: data.req.path,
+				headers: {},
+				compression: false,
+				language: null,
+				expires: true, // default policy
+			};
+
+			// add to tile stack, set primary tile
+			data.tiles.unshift(tile);
+			data.tile = tile;
+
+			next();
+
+		});
+
+	});
+
+};

package/config.dist.js

@@ -0,0 +1,153 @@
+const config = module.exports = {
+	version: 1, // config file format version
+
+	id: "tileblaster", // id of the tileblaster instance, in case you want to run more than one; default: tileblaster
+
+	threads: 1, // number of worker threads in cluster, default: 1
+	queue: 10, // number of parallel tile processes per worker, default: 12
+
+	server: {
+		url:   "https://tileserver/tiles", // public url including subdirectory
+		mount: "/tiles", // mountpoint override, default: pathname from ${config.paths.base}
+	},
+
+	paths: {
+		work:    "/path/to/stuff",   // the base directory where files (sockets, logs, plugins, ...) go; default: ~/tileblaster
+		data:    "/path/to/tiles",   // the directory in which cached tiles are saved; default: ${config.paths.work}/data
+		logs:    "/path/to/logs",    // the directory in which cached tiles are saved; default: ${config.paths.work}/logs
+		plugins: "/path/to/plugins", // the directory from which plugins ar loaded; default: ${config.paths.work}/plugins
+		sockets: "/path/to/sockets", // the directory i nwhich sockets are created; default: ${config.paths.work}/sockets
+	},
+
+	// listen
+	listen: [{
+		port: 8080, // required
+		host: "localhost", // default localhost
+	},{
+		socket: "test.socket", // required, absolute path or relative to ${config.paths.socket}
+		mode: 0o660, // change socket mode to this id
+		group: 1000, // change socket group to this is
+	}],
+
+	// plugins, relative or absolute paths for local files, npm module names otherwise
+	plugins: {
+		resize: "./resize.js", // path relative to `config.paths.plugins`, starting with ./
+		convert: "/path/to/convert.js", // absolute path
+		optimize: "someplugin", // npm module
+	},
+
+	maps: {
+		example: [{
+			// cors headers
+			// they are only useful for standalone servers
+			builtin: "cors",
+			origins: [ "https://example.org/" ],
+		},{
+			builtin: "parse", // /z/x/y@r.ext, other formats via parse function
+			parse: function(req, next){ // override parse function, req is the raw request
+
+				// req.url=/foo/bar/{z}.{x}.{y}
+				let p = req.url.split("/").pop().split(".");
+
+				// do things to get parameters from path
+				next(null, {
+					z: p[0],
+					x: p[1],
+					y: p[2],
+					r: "",
+					w: 256,
+					d: 1,
+					e: ".png"
+				});
+			},
+		},{
+			builtin: "check",
+			zoom: [ 0, 22 ], // min, max
+			bbox: [ -180, -90, 180, 90 ], // west, south, east, north
+			extensions: [ "png", "jpeg" ], // allowed extensions
+			density: [ "", "@2x", "@3x" ], // allowed density markeers
+			check: function(params, fn) { // override check function, params from parse
+				fn(new Error("Check failed")); // deliver error if check failed
+			}
+		},{ // get from cache, skip to `skipto` if successful
+			builtin: "cache",
+			skipto: "deliver",
+		},{
+			builtin: "noop", // does nothing
+		},{
+			builtin: "tileserver",
+			url: "https://{s}.tileserver.example/test/{z}/{x}/{y}{r}.{e}",
+			subdomains: [ "a", "b", "x" ], // random chosen and passed in as {s}
+			tms: true, // y coordinate is inverted
+			headers: {}, // additional headers sent to the backend tileserver
+			status: [ 200 ], // expected status code(s)
+			mimetypes: [ "image/png", "image/jpeg" ], // expected mime types
+			mimetype: "image/png", // overwrite mime type from server
+		},
+		/* alternative:
+		{
+			// get tile from versatiles container
+			builtin: "versatiles",
+			url: "https://cdn.example/planet.versatiles",
+			headers: { // headers sent to versatiles server
+				"X-Tileblaster": "True",
+			},
+		},{
+			// get tiles from pmtiles container
+			builtin: "pmtiles",
+			url: "https://cdn.example/planet.pmtiles"
+		},{
+			// get tiles from local mbtiles database
+			builtin: "mbtiles",
+			file: "/path/to/planet.mbtiles"
+		},
+		*/
+		{
+			// edit vectortile
+			builtin: "edit",
+			edit: function(layers){
+
+				// remove unused layer
+				layers = layers.filter(function(layer){
+					return (layer.name !== "unused-layer");
+				});
+
+				return layers;
+			}
+		},{
+			// use sharp for image manipulation
+			plugin: "sharp",
+			resize: { width: 512, height: 512 }, // sharp.resize()
+		},{
+			plugin: "optimize",
+			png: { o: 4 }, // true or opts for optipng
+			jpeg: true, // true or opts for mozjpeg
+		},{
+			// convert raster tiles to webp and/or avif
+			builtin: "modernize",
+			webp: {
+				quality: 90,
+				effort: 4,
+			},
+			avif: {
+				quality: 90,
+				effort: 5,
+			},
+		},{
+			builtin: "compress",
+			brotli: 8, // true or <level> or {opts}
+			gzip: true, // true or <level> or {opts}
+		},{
+			builtin: "cache",
+			expires: "30d",
+		},{
+			// debug output
+			builtin: "dump",
+		},{
+			builtin: "deliver", // deliver best matching tile for client
+			headers: {}, // additional http headers
+		}],
+		// more maps
+		othermap: [],
+	}
+};