tileblaster 0.4.9 → 1.0.0

package/lib/purge.js

@@ -0,0 +1,109 @@
+// purge cache cache
+
+const worker = require("node:worker_threads");
+
+if (worker.isMainThread) {
+
+	const debug = require("./debug");
+
+	const purge = module.exports = function purge(){
+
+		// create cleanup worker
+		const debug = require("./debug");
+		debug.info("Creating Purge Worker");
+		new worker.Worker(__filename, {
+			argv: process.argv.slice(2),
+			env: process.env
+		}).on("message", function(message){
+			if (message.hasOwnProperty("purged")) debug.info("Purged %d files", message.purged);
+		}).unref();
+
+	};
+
+	// run cleanup if called directly
+	if (require.main === module) purge();
+
+} else { // cleanup worker
+
+	const dur = require("dur");
+	const config = require("./config");
+	const klaw = require("klaw");
+	const path = require("node:path");
+	const fs = require("node:fs");
+
+	// determine paths
+	if (!config.paths) config.paths = {};
+	if (!config.paths.work) config.paths.work = path.resolve(os.homedir(), "tileblaster");
+	if (!config.paths.data) config.paths.data = path.resolve(config.paths.work, "data");
+
+	const caches = Object.entries(config.maps).reduce(function(caches,[map,tasks]){
+
+		// find cache config fore map
+		let cache = tasks.find(function(task){
+			return task.builtin === "cache" && task.hasOwnProperty("expires");
+		});
+
+		// no cache here
+		if (!cache) return caches;
+
+		// parse expires, lifted from the builtin
+		let expires;
+		switch (typeof cache.expires) {
+			case "boolean":
+				expires = cache.expires;
+			break;
+			case "number":
+				expires = ((Number.isFinite(cache.expires)) ? Math.max(0,Math.round(cache.expires*1000)) : (cache.expires > 0)) || true;
+			break;
+			case "string":
+				expires = dur(cache.expires, true);
+			break;
+			default:
+				expires = true; // expires immediately
+			break;
+		}
+
+		if (!expires) return caches;
+		if (expires === true) expires = 0;
+
+		caches.push({
+			dir: path.join(config.paths.data, map),
+			expires: Date.now()+expires,
+		});
+
+		return caches;
+
+	},[]);
+
+	if (caches.length === 0) {
+		worker.parentPort.postMessage({ purged: 0 });
+	} else {
+
+		// create tasks
+		let purged = 0;
+		Promise.allSettled(caches.map(function(cache){
+			return new Promise(function(resolve,reject){
+
+				let deletable = [];
+
+				klaw(cache.dir).on("data", function(file){
+					if (file.stats.mtimeMs < cache.expires) deletable.push(file.path);
+				}).on("end", function(){
+					Promise.allSettled(deletable.map(function(file){
+						return new Promise(function(resolve,reject){
+							fs.unlink(file, function(err){
+								if (err) return reject(err);
+								purged++;
+								resolve();
+							});
+						});
+					})).then(resolve).catch(reject);
+				});
+			});
+		})).then(function(){
+			worker.parentPort.postMessage({ purged });
+		});
+
+	};
+
+};

package/lib/retrieve.js

@@ -0,0 +1,15 @@
+const phin = require("phin");
+
+// prepare agents with keepalive enabled
+const agents = {
+	http: new require("node:http").Agent({ keepAlive: true }),
+	https: new require("node:https").Agent({ keepAlive: true }),
+};
+
+// agentify phin
+const retrieve = module.exports = async function retrieve(opts, fn) {
+	if (typeof opts === "string") opts = { url: opts }; // ensure opts is object
+	if (!opts.hasOwnProperty("core")) opts.core = {}; // ensure core is present
+	if (!opts.core.hasOwnProperty("agent")) opts.core.agent = agents[opts.url.slice(0,opts.url.indexOf(":"))]; // set agent by protocol
+	return phin(opts, fn);
+};

package/lib/rfc822date.js

@@ -0,0 +1,9 @@
+// convert a date object into a rfc 822 date
+
+const months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
+const days = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
+
+module.exports = function(date){
+	let tzo = date.getTimezoneOffset();
+	return days[date.getDay()] + ", " + date.getDate().toString().padStart(2,"0") + " " + months[date.getMonth()] + " " + date.getFullYear() + " " + date.getHours().toString().padStart(2,"0") + ":" + date.getMinutes().toString().padStart(2,"0") + ":" + date.getSeconds().toString().padStart(2,"0") + " " + (tzo > 0 ? "-" : "+") + Math.abs(Math.floor(tzo/60)).toString().padStart(2,"0") + Math.abs(tzo%60).toString().padStart(2,"0");
+};

package/lib/router.js

@@ -0,0 +1,95 @@
+const normalize = require("node:path").normalize;
+const tasks = require("./tasks");
+
+// unfancy router, good enough for tileblaster
+const router = module.exports = function router({ mountpoint }) {
+	if (!(this instanceof router)) return new router(...arguments);
+	const self = this;
+
+	self.routes = {};
+	self.uses = [];
+
+	// mountpoint, without trailing slash
+	self.mountpoint = self.fixpath(mountpoint || "");
+
+	// default default route, very bare bones
+	self.routes[""] = function(req, res){
+		res.statusCode = 404, res.end();
+	};
+
+	return self;
+};
+
+// route handler, pass to server
+router.prototype.serve = function(req, res) {
+	const self = this
+
+	// prepare
+	let path = req.url
+
+	// if url get path only
+	if (path.charCodeAt(0) !== 47) path = path.replace(/^https?:\/\/.*?\//,"/");
+
+	// trim query string and fragment
+	if (path.includes("?")) path = path.slice(0, path.indexOf("?"));
+	if (path.includes("#")) path = path.slice(0, path.indexOf("#"));
+
+	// trim mountpoint
+	if (self.mountpoint && path.slice(0,self.mountpoint.length) === self.mountpoint) path = path.slice(self.mountpoint.length);
+
+	// fix path
+	path = self.fixpath(path);
+
+	// FIXME some header prep? other things?
+
+	// find route
+	let route = path;
+	// while (!self.routes.hasOwnProperty(route)) route = route.slice(0, route.lastIndexOf("/")); // in case we need it later
+	if (!self.routes.hasOwnProperty(route)) route = ""; // good for now
+
+	// apply to req
+	req.path = path;
+	req.route = route;
+
+	// apply uses
+	tasks(self.uses).run(req, res, function(err, req, res){
+
+		// very unglamourous server error
+		if (err) return res.statusCode = 500, res.end();
+
+		// call route
+		self.routes[req.route](req, res);
+
+	});
+
+};
+
+// add route
+router.prototype.route = function(path, route) { // function(req, res){}
+	return this.routes[path] = route, this;
+};
+
+// add middleware
+router.prototype.use = function(use) { // function(req, res, next){}
+	return this.uses.push(use), this;
+};
+
+// set default route
+router.prototype.default = function(route) {
+	return this.routes[""] = route, this;
+};
+
+// fix path
+router.prototype.fixpath = function(path) {
+
+	// ensure leadin slash
+	if (path.charCodeAt(0) !== 47) path = "/"+path;
+
+	// remove trailing slashes
+	while (path.length > 1 && path.charCodeAt(path.length-1) === 47) path = path.slice(0, -1);
+
+	// normalize path
+	path = normalize(path);
+
+	return path;
+};

package/lib/store.js

@@ -0,0 +1,161 @@
+// store files to disk, use xattr to keep metadata
+
+const fs = require("node:fs");
+const dur = require("dur");
+const path = require("node:path");
+const xattr = require("fs-xattr");
+const rfc822date = require("./rfc822date");
+
+const store = module.exports = function store({ root }){
+	if (!(this instanceof store)) return new store(...arguments);
+
+	this.root = path.resolve(process.cwd(), root);
+	this.cache = {}; // FIXME use lru cache
+
+	return this;
+};
+
+// find best file on disk
+store.prototype.find = function(file, extensions, fn){
+	const self = this;
+
+	// find all files in question
+	Promise.allSettled([ "", ...extensions.map(function(e){
+		return e[0] === "." ? e : "."+e;
+	})].map(function(ext){
+		return new Promise(function(resolve, reject){
+			const filepath = path.join(self.root, file+ext);
+			fs.stat(filepath, function(err, stats){
+				if (err) reject();
+				// get extended attributes
+				xattr.get(filepath, "user.tileblaster").then(function(attr){
+
+					try {
+						attr = JSON.parse(attr);
+					} catch (err) {
+						return reject(err);
+					}
+
+					// check if expired
+					if (attr.expires && (attr.expires === true || attr.expires < Date.now())) return reject();
+
+					resolve({
+						file: filepath,
+						stats: stats,
+						attr: attr,
+					});
+
+				}).catch(function(err){
+					reject(err);
+				});
+			})
+		});
+	})).then(function(found){
+
+		// extract
+		found = found.filter(function(f){
+			return f.status === "fulfilled";
+		}).map(function(f){
+			return f.value;
+		})
+
+		switch (found.length) {
+			case 0: return fn(null, null); break;
+			case 1: return fn(null, found[0]); break;
+			default:
+				// find the smallest size
+				return fn(null, Array.from(found).sort(function(a,b){
+					return a.stats.size - b.stats.size;
+				}).shift());
+			break;
+		}
+	});
+
+	return this;
+};
+
+// store file and attributes to disk
+store.prototype.put = function(tile, fn){
+	const self = this;
+
+	const destfile = path.join(self.root, tile.path);
+	const tmpfile = destfile+".tmp";
+
+	// check if exists and still valid?
+	self.check(destfile, function(err, isValid){
+		if (!err && isValid) return fn(null); // end if file is still valid
+
+		// ensure dest dir
+		fs.mkdir(path.dirname(tmpfile), { recursive: true }, function(err){
+			if (err) return fn(err);
+
+			// write tmp file
+			fs.writeFile(tmpfile, tile.buffer, function(err){
+				if (err) return fn(err);
+
+				// set etag and rfc822-date
+				fs.stat(tmpfile, function(err, stats){
+					if (err) return fn(err);
+
+					tile.headers.etag = '"'+Math.floor(stats.mtimeMs/1000).toString(16)+'-'+(stats.size).toString(16)+'"';
+					tile.headers["last-modified"] = rfc822date(stats.mtime);
+					tile.headers["expires"] = rfc822date(new Date(tile.expires));
+
+					// store anything but buffer in xattr
+					const attr = Object.entries(tile).reduce(function(attr, [ k, v ]){
+						if (k !== "buffer") attr[k] = v;
+						return attr;
+					},{});
+
+					// set attributes
+					xattr.set(tmpfile, "user.tileblaster", JSON.stringify(attr)).then(function(){
+
+						// switch tmp → file
+						fs.rename(tmpfile, destfile, function(err){
+							if (err) return fn(err), fs.unlink(tmpfile, function(){}); // attempt unlinking, no feedback
+							fn(null);
+
+						});
+
+					}).catch(function(err){
+						return fn(err);
+					});
+
+				});
+
+			});
+
+		});
+
+	});
+
+	return this;
+};
+
+store.prototype.check = function(file, fn) { // err, isValid
+	const self = this;
+
+	fs.stat(file, function(err, stat){
+		if (err && err.code === "ENOENT") return fn(null, false); // does not exist
+		if (err) return fn(err);
+
+		xattr.get(file, "user.tileblaster").then(function(attr){
+
+			// try parse xattr
+			try {
+				attr = JSON.parse(attr);
+			} catch (err) {
+				return fn(err);
+			}
+
+			// file is valid when no expires property or expires < now
+			return fn(null, (!attr.expires || attr.expires < Date.now()));
+
+		}).catch(function(err){
+			return fn(err);
+		});
+
+	});
+
+	return this;
+};

package/lib/strtpl.js

@@ -0,0 +1,15 @@
+// minimalistic string tempate, no regex
+const types = { string: true, number: true, boolean: true };
+const strtpl = module.exports = function strtpl(str, params){
+	let out = "", j = 0;
+	for (let i = 0; i < str.length; i++) {
+		j = i+1;
+		if (str[i] === "{" && str[i+2] === "}" && str[j] >= "a" && str[j] <= "z") {
+			if (types[typeof params[str[j]]]) out += params[str[j]];
+			i+= 2;
+		} else {
+			out += str[i];
+		}
+	}
+	return out;
+};

package/lib/tasks.js

@@ -0,0 +1,50 @@
+// recursive task queue
+const tasks = module.exports = function tasks(jobs){
+	if (!(this instanceof tasks)) return new tasks(...arguments);
+	const self = this;
+	self.stack = [];
+	if (jobs) self.push(jobs);
+	return this;
+};
+
+// add jobs to stack
+tasks.prototype.push = function(jobs){
+	const self = this;
+	if (!Array.isArray(jobs)) jobs = [ jobs ];
+	jobs.forEach(function(fn){
+		if (typeof fn === "function") self.stack.push(fn);
+	});
+	return this;
+};
+
+// recursively run stack until empty
+tasks.prototype.run = function(){
+	const self = this;
+	let args = Array.from(arguments);
+	let fn = args.pop(); // callback should always be last argument
+	if (typeof fn !== "function") throw new Error("tasks.run needs a function argument");
+	if (typeof args === "function") fn = args, args = {};
+	if (self.stack.length === 0) return fn(null, ...args);
+	try {
+		self.stack.shift()(...args, function(err){
+			if (err) return fn(err, ...args);
+			return self.run(...args, fn);
+		}, function(label){ // skip
+			if (label) {
+				// skip stack until found
+				let nextlabel = self.stack.findIndex((f)=>(f.label === label));
+				if (nextlabel >= 0) {
+					self.stack = self.stack.slice(nextlabel);
+					return self.run(...args, fn);
+				} else {
+					return fn(new Error("Tasks: Unable to find '"+label+"'"), ...args);
+				}
+			} else {
+				return fn(null, ...args); // end execution
+			}
+		});
+	} catch (err) {
+		return fn(err, ...args);
+	}
+	return this;
+};

package/package.json

@@ -1,11 +1,14 @@
 {
   "name": "tileblaster",
-  "version": "0.4.9",
-  "description": "pretty fast optimizing & compressing tile caching proxy",
-  "main": "lib/tileblaster.js",
+  "version": "1.0.0",
+  "description": "a quick and versatile map tile caching proxy",
+  "main": "tileblaster.js",
   "bin": {
     "tileblaster": "bin/tileblaster.js"
   },
+  "scripts": {
+    "dev": "DEBUG=tileblaster node bin/tileblaster-dev.js"
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/yetzt/tileblaster.git"
@@ -17,25 +20,34 @@
   },
   "homepage": "https://github.com/yetzt/tileblaster#readme",
   "dependencies": {
-    "dur": "^0.0.3",
-    "klaw": "^3.0.0",
+    "colrz": "^0.0.5",
+    "dur": "^1.0.1",
+    "fs-xattr": "^0.3.1",
+    "klaw": "^4.1.0",
     "minimist": "^1.2.8",
     "node-watch": "^0.7.3",
     "phin": "^3.7.0",
     "quu": "^0.4.3"
   },
   "optionalDependencies": {
-    "debug": "^4.3.4",
-    "jpck": "^1.0.2",
-    "node-zopfli": "^2.1.4",
-    "nsa": "^0.2",
-    "pnck": "^1.0.1",
-    "versatiles": "^0.3.0"
+    "fzstd": "^0.1.0",
+    "js-mozjpeg": "^0.1.2",
+    "mbg": "^0.0.2",
+    "node-liblzma": "^1.1.9",
+    "optipng-js": "^0.1.2",
+    "pmtiles": "^2.7.1",
+    "sharp": "^0.32.0",
+    "versatiles": "^0.3.1",
+    "vtt": "^0.0.3"
   },
   "engines": {
-    "node": ">= 10"
+    "node": ">=14"
   },
   "os": [
-    "!win32"
+    "darwin",
+    "linux"
+  ],
+  "keywords": [
+    "tileblaster", "tileserver", "map", "tiles", "tile", "mbtiles", "versatiles", "pmtiles", "server", "proxy", "cache", "vectortiles", "rastertiles", "mvt", "pbf"
   ]
 }

package/plugins/example.js

@@ -0,0 +1,22 @@
+// bare bones example plugin
+module.exports = function({ req, res, opts, data }, next, skip){
+
+	// next() calls the next task
+	// skip(name) skips all the next tasks until a plugin or builtin calles `name`, or until the end
+
+	// req and res are your http connection.
+	// if you consume res (e.g. send a resonse and call `res.edn()`),
+	// then set `res.used = true;` and use skip() to end processing
+
+	// opts contains everything from the config
+
+	// data contains all the data that's passed along
+	// data.tile for the "main" tile
+	// data.tiles[] for alternative versions
+
+	// let's so something:
+	this.lib.debug.info("I'm a plugin!");
+
+	// continue
+	next();
+};

package/readme.md

@@ -1,58 +1,91 @@
 # tileblaster
 
-tileblaster is a map tile caching (and optimizing) proxy, designed to run with nginx.
+![tileblaster](docs/tileblaster.png)
 
-## install
+tileblaster is a versatile caching proxy server for map tiles. it can handle many different tile sources and file formats
+and can optimise tiles on the fly and speed up delivery by acting as a cache.
 
-`npm i tileblaster -g`
+## Awesome things you can do with tileblaster
 
-use `--no-optional` if you don't want tile optimization or [versatiles](https://github.com/versatiles-org/versatiles-spec) support.
+* Serve tiles from any ZXY/TMS tileserver, [VersaTiles container](https://versatiles.org/), pmtiles container or mbtiles database.
+* Edit vector tiles on the fly with [Vector Tile Transformer](https://www.npmjs.com/package/vtt)
+* Optimize raster tiles with `mozjpeg` / `optipng`, convert them to `webp` / `aviv` format on the fly or edit them with [sharp](https://www.npmjs.com/package/sharp).
+* Precompress tiles with `gzip` and `brotli`
+* Cache remote files locally
 
-## run
+and much more
 
-`tileblaster /path/to/config.js`
+## What tileblaster isn't
 
-Use [pm2](https://npmjs.com/package/pm2), [nodemon](https://npmjs.com/package/nodemon), [forever](https://npmjs.com/package/forever) or similar to run tileblaster as service;
+tileblaster is not a tileserver, it does not read raw OpenStreetMap data or create map tiles from scratch; you need to
+have a source for map tiles. You can of course use tools like [tilemaker](https://tilemaker.org/) to create your own
+tilesets, use freely available ready-made tiles from [Versatiles](https://versatiles.org/) or use another tileserver
+if you're allowed to do so.
 
-## configuration
+## Install
 
-see [config.js.dist](config.js.dist)
+`npm i -g tileblaster`
 
-### nginx configuration
+## Usage
+
+`tileblaster [options] [-c] config.js`
+
+### Options
+
+* `-c` `--config <config.js>` - load config file
+* `-p` `--port <[host:]port>` - listen on this port (overrides config)
+* `-s` `--socket <socket[,mode,gid]>` -  on this socket (overrides config)
+* `-t` `--threads <num>` - number of threads (overrides config)
+* `-h` `--help` - print help screen
+* `-v` `--verbose` - enable debug output
+* `-q` `--quiet` - disable debug output
+
+## Configuration
+
+See [Configuration](docs/config.md) and [Examples](docs/examples.md)
+
+### Plugins
+
+tileblaster supports plugins. They work just like builtins, but you can specify them in `config.plugins`
+
+[Example Plugin](plugins/example.js)
+
+### Nginx
+
+tileblaster is easy to use with nginx acting as a reverse proxy. Here is a simple example:
 
 ```
-upstream upstream_tileblaster {
-	server unix:/path/to/tileblaster.sock;
+upstream tileblaster {
+	server 127.0.0.1:28897;
+	# server unix:/path/to/tileblaster.socket; # ← if you use sockets
 }
 
 server {
-	listen 80;
-	server_name tileblaster;
-
-	gzip_static on;
-	# brotli_static on; # if ngx_brotli is available
-
-	if (-f $document_root/$uri.err) {
-		return 204;
-	}
 
-	location / {
-		root /path/to/tileblaster/tiles;
-		try_files $uri $uri/ @tileblaster;
+	# ...
+
+	location /tileblaster { # ← set config.server.mount to the same path
+		proxy_set_header Host $host;
+		proxy_set_header Origin $http_origin;
+		proxy_set_header Accept-Encoding $http_accept_encoding;
+		proxy_set_header Accept-Language $http_accept_language;
+		proxy_set_header Accept $http_accept;
+		proxy_set_header If-Modified-Since $http_if_modified_since
+		proxy_set_header If-None-Match $http_if_none_match
+		proxy_http_version 1.1;
+		proxy_pass http://tileblaster;
 	}
 
-	location @tileblaster {
-		proxy_pass http://upstream_tileblaster;
-	}
 }
+
 ```
 
-## usage
+## Optional Dependencies
+
+tileblaster has a few optional dependencies, that are mostly used for image manilulation and optimisation (Sharp, MozJPEG, OptiPNG)  or more complex tile sources (Versatiles, PMTiles, MBTiles).
 
-get the tiles via `http://server/<mapid>/<z>/<x>/<y>[<d>].<ext>`
+If you don't need them, install tileblaster with `npm i -g tileblaster --no-optional`
 
-* `<mapid>` is the map id specified in your `config.js`
-* `<z>`, `<x>` and `<z>` are the tile coorinates
-* `<d>` is the optional pixel density marker, for example `@2x`
-* `<ext>` is the extension, for example `png`, `geojson` or `pbf`
+## License
 
+[Unlicense](./UNLICENSE.md)