tileblaster 0.2.6 → 0.3.2

package/bin/tileblaster.js

@@ -1,14 +1,25 @@
 #!/usr/bin/env node
 var argv = require('minimist')(process.argv.slice(2));
-require("../lib/tileblaster.js")(
+var watch = require('node-watch');
+var configfile = require("path").resolve.apply(global, (!!argv._[0]) ? [ process.cwd(), argv._[0] ] : [ "../config.js" ]);
+var tb = require("../lib/tileblaster.js")(
 	(function(config){
 		["socket","tiles","queue","id"].forEach(function(n){
 			if (!!argv[n]) config[n] = argv[n];
 		});
+		// watch changes in config file
+		watch(configfile, function(evt,f){
+			if (evt === "update") try {
+				delete require.cache[require.resolve(configfile)];
+				tb.reconfigure(require(configfile));
+			} catch (err) {
+				console.error("Unable to read changed config file", err);
+			}
+		});
 		return config;
 	})((function(){
 		try {
-			return (!!process.argv[2]) ? require(require("path").resolve(process.cwd(), process.argv[2])) : require("../config.js");
+			return require(configfile);
 		} catch (err) {
 			console.error("usage: tileblaster <config.js> [--socket tileblaster.sock] [--tiles /path/to/tiles] [--queue 100] [--id mytileblaster]");
 			process.exit(1);

package/config.js.dist

@@ -56,6 +56,12 @@ module.exports = {
 			// * .png with `optipng`
 			// * .jpg with `mozjpeg`
 			"optimize": true,
+			
+			// compress tiles (only makes sense for non-rastered tiles like json, pbf, mvt)
+			"compress": [ "gz", "br" ],
+			
+			// write tiles to disk
+			"cache": true,
 
 			// minimum time tiles are kept
 			"expires": "1d",

package/lib/tileblaster.js

@@ -5,6 +5,7 @@ var url = require("url");
 var path = require("path");
 var http = require("http");
 var util = require("util");
+var zlib = require("zlib");
 var stream = require("stream");
 
 var debug = require("debug")("tileblaster");
@@ -17,6 +18,7 @@ var dur = require("dur");
 // optional dependencies; if only 
 try { var pnck = require("pnck"); } catch (err) { var pnck = null; }
 try { var jpck = require("jpck"); } catch (err) { var jpck = null; }
+try { var zopfli = require("node-zopfli"); } catch (err) { var zopfli = null; }
 
 // load package
 var pckg = require("../package.json");
@@ -47,6 +49,9 @@ tileblaster.prototype.mime = {
 	arcamf:   'application/octet-stream',
 };
 
+// compression formats
+tileblaster.prototype.comp = [ "gz", "br" ];
+
 // initialize
 tileblaster.prototype.init = function(config){
 	var self = this;
@@ -56,6 +61,36 @@ tileblaster.prototype.init = function(config){
 	// server
 	self.srvr = null;
 
+	// statistics
+	self.statistics = { 
+		hits: 0, 
+		phits: 0,
+		served: 0, 
+		last: Date.now() 
+	};
+
+	// compression queue
+	self.cqueue = queue(1);
+	
+	// have maps with active expires?
+	self.expiration = false;
+
+	// reconfigure
+	self.reconfigure(config);
+
+	return this;
+	
+};
+
+// reconfigure
+tileblaster.prototype.reconfigure = function(config){
+	var self = this;
+
+	debug("<init> reconfiguring");
+	
+	// cache
+	self.errcache = {};
+	
 	// keep config
 	self.config = config || {};
 	
@@ -74,26 +109,12 @@ tileblaster.prototype.init = function(config){
 	// queue size
 	self.config.queue = Math.max(parseInt(self.config.queue,10)||100,1);
 
-	// cache
-	self.errcache = {};
-	
-	// statistics
-	self.statistics = { 
-		hits: 0, 
-		phits: 0,
-		served: 0, 
-		last: Date.now() 
-	};
-
 	// extend mime types
 	if (!!self.config.mime) Object.keys(self.config.mime).map(function(ext){ self.mime[ext] = self.config.mime[ext] });
 	
 	// queue
 	self.queue = queue(self.config.queue);
-	
-	// have maps with active expires?
-	self.expiration = false;
-	
+
 	// check configured maps
 	self.maps = Object.keys(self.config.maps).reduce(function(maps, id){
 	
@@ -112,10 +133,10 @@ tileblaster.prototype.init = function(config){
 			}
 			
 			// resolutions
-			if (!map.res) map.res = [];
+			if (!map.res) map.res = [ "" ];
 			if (!(map.res instanceof Array)) map.res = [ map.res ];
 			map.res = map.res.filter(function(res){
-				return /^@[1-9][0-9]*(\.[0-9]+)?x$/.test(res);
+				return (res === "") || /^@[1-9][0-9]*(\.[0-9]+)?x$/.test(res);
 			});
 		
 			// precalculate bbox for zoom levels
@@ -151,6 +172,13 @@ tileblaster.prototype.init = function(config){
 			// expires
 			if (!!map.expires) map.expires = dur(map.expires);
 			if (!!map.expires) self.expiration = true;
+			if (!map.expires) map.expires = Infinity;
+			
+			// cache
+			map.cache = (!map.hasOwnProperty("cache")) ? true : (!!map.cache);
+			
+			// compression
+			map.compress = map.compress.filter(function(c){ return (self.comp.indexOf(c) >= 0) });
 			
 			return map;
 			
@@ -159,13 +187,13 @@ tileblaster.prototype.init = function(config){
 	},{});
 	
 	// build user agent
-	if (!self.config.useragent) self.config.useragent = util.format("%s/%s (+)", pckg.name, pckg.version, pckg.homepage);
+	if (!self.config.useragent) self.config.useragent = util.format("%s/%s (+%s)", pckg.name, pckg.version, pckg.homepage);
 
 	// cleanup timer
 	self.config.cleanup = (!!self.config.cleanup) ? dur(self.config.cleanup) : false;
+	if (!!self.cleaner) clearInterval(self.cleaner);
 	if (self.config.cleanup && self.expiration) self.cleaner = setInterval(function(){ self.cleanup(); }, self.config.cleanup).unref();
-
-	return this;
+	
 };
 
 // return server
@@ -179,32 +207,40 @@ tileblaster.prototype.server = function(){
 		// check http method
 		if (req.method !== "GET") return debug("<server> invalid method: %s", req.method), (!!self.config.hints&&res.setHeader("x-err-hint","invalid method")), res.statusCode = 405, res.end();
 
-		// only tile requests are allowed FIXME optimize
-		if (!(/\/(([a-z0-9\-\_\.]+)\/([0-9]+)\/([0-9]+)\/([0-9]+)(@[0-9]+(\.[0-9]+)?x)?\.([a-z0-9]+))$/.exec(url.parse(req.url).pathname.toLowerCase()))) return debug("<server> invalid request: %s", url.parse(req.url).pathname), (!!self.config.hints&&res.setHeader("x-err-hint","invalid request")), res.statusCode = 404, res.end();
+		// parse request
+		var t = self._tilepath(url.parse(req.url).pathname.toLowerCase());
+		if (!t) return debug("<server> invalid request: %s", url.parse(req.url).pathname), (!!self.config.hints&&res.setHeader("x-err-hint","invalid request")), res.statusCode = 404, res.end();
 
-		// asselmble information
-		var p = RegExp.$1;
-		var mapid = RegExp.$2;
-		
 		// check if map exists
-		if (!self.maps.hasOwnProperty(mapid)) return debug("<server> requested invalid map: %s", mapid), (!!self.config.hints&&res.setHeader("x-err-hint","invalid map")), res.statusCode = 404, res.end();
+		if (!self.maps.hasOwnProperty(t.mapid)) return debug("<server> requested invalid map: %s", t.mapid), (!!self.config.hints&&res.setHeader("x-err-hint","invalid map")), res.statusCode = 404, res.end();
+
+		// deliver file if requested
+		if (t.type === "file") {
+			switch (t.file) {
+				case "tile.json":
+					res.writeHead(200, { "Content-Type": "application/json" });
+					res.end(self._tilejson(t.mapid, ((req.headers["x-https"] === "on") ? "https://" : "http://")+req.headers["host"]));
+				break;
+				default:
+					return debug("<server> requested invalid file: %s", t.p), (!!self.config.hints&&res.setHeader("x-err-hint","invalid file")), res.statusCode = 404, res.end();
+				break;
+			}
+			return;
+		}
 
-		var z = parseInt(RegExp.$3,10);
-		var x = parseInt(RegExp.$4,10);
-		var y = parseInt(RegExp.$5,10);
-		var r = ((self.maps[mapid].res.indexOf(RegExp.$6) >= 0) ? RegExp.$6 : false);
-		var ext = RegExp.$8;
+		// res
+		t.r = (!!t.r && (self.maps[t.mapid].res.indexOf(t.r) >= 0) ? t.r : false);
 
-		debug("<server> [%s] requested", p);
+		debug("<server> [%s] requested", t.p);
 
-		self.tile(mapid, z, x, y, r, ext, function(err, stream, meta){
-			if (err) return debug("<server> [%s] error: %s", p, err), (!!self.config.hints&&res.setHeader("x-err-hint",err.toString())), res.statusCode = 204, res.end();
+		self.tile(t.mapid, t.z, t.x, t.y, t.r, t.ext, function(err, stream, meta){
+			if (err) return debug("<server> [%s] error: %s", t.p, err.toString()), (!!self.config.hints&&res.setHeader("x-err-hint",err.toString())), res.statusCode = 204, res.end();
 			
 			// send headers
 			res.writeHead(200, { "Content-Type": meta['content-type'] });
 			
 			stream.on("end", function(){
-				debug("<server> [%s] done", p);
+				debug("<server> [%s] done", t.p);
 				self.statistics.served++;
 			});
 			
@@ -300,8 +336,16 @@ tileblaster.prototype.heartbeat = function(){
 // delete expired tiles
 tileblaster.prototype.cleanup = function(){
 	var self = this;
+
+	// clean up error cache
+	var d = Date.now();
+	self.errcache = self.errcache.filter(function(e){
+		return (e.until > d);
+	});
+	
+	// FIXME: extra process
 	
-	// 
+	// clean up map tiles
 	Object.keys(self.maps).filter(function(mapid){ 
 		return (!!self.maps[mapid].expires);
 	}).forEach(function(mapid){
@@ -357,7 +401,7 @@ tileblaster.prototype.tile = function(mapid, z, x, y, r, e, fn){
 		debug("<tile> [%s] valid", tilefile);
 	
 		// check for cached 404 tiles
-		if (!!self.errcache[tilefile] && self.errcache[tilefile] > (Date.now()-self.config.expires)) return fn(new Error("Known bad tile"), null);
+		if (!!self.errcache[tilefile] && (self.errcache[tilefile].until > Date.now())) return fn(new Error("Known bad tile: "+self.errcache[tilefile].err), null);
 	
 		// resolve tile path
 		var tilepath = path.resolve(self.config.tiles, tilefile);
@@ -367,23 +411,31 @@ tileblaster.prototype.tile = function(mapid, z, x, y, r, e, fn){
 
 		self.fetchtile(tileurl, mapid, function(err, stream, meta){
 
-			// locally cache 404 and 204
-			if (!!err && (stream === 404 || stream === 204)) self.errcache[tilefile] = Date.now();
+			if (err) {
+				if (stream !== null) {
+					// cache error tile
+					self.errcache[tilefile] = { until: (Date.now()+self.config.maps[mapid].expires), err: err, code: stream };
 
-			// error or no stream
-			if (!!err || !stream || typeof stream === 'number') return fn(err, stream);
+					// mark tile as erronous in file system
+					fs.writeFile(tilepath+".err", JSON.stringify(self.errcache[tilefile]), function(err){
+						if (err) return debug("<tile> [%s.err] error: %s", tilefile, err.toString());
+					});
+				}
+				return debug("<tile> [%s] error: %s", tilefile, err.toString()), fn(err, stream);
+			}
 
 			debug("<tile> [%s] fetched", tilefile);
 
-			// strem mux
-			stream.pipe(self.optimize(mapid, e)).pipe(self._mux(function(stream){
+			var streams = [];
 
+			streams.push(function(stream){
 				// call back with stream
 				fn(null, stream, {
 					'content-type': (self.mime[e])
 				});
-
-			}, function(stream){
+			});
+			
+			if (!!self.config.maps[mapid].cache) streams.push(function(stream){
 				
 				// don't overwrite if tile exists
 				fs.access(tilepath, fs.constants.F_OK, function(err){
@@ -394,6 +446,10 @@ tileblaster.prototype.tile = function(mapid, z, x, y, r, e, fn){
 						// save to tmp file, rename when done
 						stream.pipe(fs.createWriteStream(tilepath+".tmp").on('finish', function(){
 							fs.rename(tilepath+".tmp", tilepath, function(){
+								
+								// compress
+								if (self.config.maps[mapid].compress instanceof Array && self.config.maps[mapid].compress.length > 0) self.compress(tilepath, self.config.maps[mapid].compress);
+								
 								debug("<tile> [%s] saved", tilefile);
 							});
 						})); 
@@ -404,8 +460,10 @@ tileblaster.prototype.tile = function(mapid, z, x, y, r, e, fn){
 
 				});
 
+			});
 
-			}));
+			// stream mux
+			stream.pipe(self.optimize(mapid, e)).pipe(self._mux.apply(self, streams));
 
 		});
 		
@@ -434,7 +492,7 @@ tileblaster.prototype.fetchtile = function(tileurl, mapid, fn){
 			// check mime type, status code, content-length, FIXME: cache!
 			if (resp.statusCode !== 200) return fn(new Error("status code "+resp.statusCode), resp.statusCode);
 			if (!resp.headers['content-type']||(!!self.maps[mapid].mime&&self.maps[mapid].mime.indexOf(resp.headers['content-type'])<0)) return fn(new Error("invalid content type "+resp.headers['content-type']), resp.statusCode);
-			if (!!resp.headers['content-length']&&parseInt(resp.headers['content-type'],10)===0) return fn(new Error("no content"), resp.statusCode);
+			if (!!resp.headers['content-length']&&parseInt(resp.headers['content-length'],10)===0) return fn(new Error("no content"), resp.statusCode);
 
 			// signal queue when read stream has finished
 			this.once('end', function(){ (!d++)&&done(); });
@@ -444,7 +502,7 @@ tileblaster.prototype.fetchtile = function(tileurl, mapid, fn){
 			return fn(null, this, {
 				date: (new Date(resp.headers.date||Date.now()).valueOf()),
 				size: (parseInt(resp.headers['content-length'],10)||null),
-				mime: (resp.headers['content-type']||'apllication/octet-stream'),
+				mime: (resp.headers['content-type']||'application/octet-stream'),
 			});
 	
 		}).on('error', function(err){
@@ -485,6 +543,54 @@ tileblaster.prototype.optimize = function(mapid, ext){
 		
 };
 
+// compression
+tileblaster.prototype.compress = function(file, comp){
+	var self = this;
+	comp.forEach(function(c){
+		self.cqueue.push(function(done){
+			switch (c) {
+				case "br":
+					// use builtin brotli
+					fs.createReadStream(file).pipe(zlib.createBrotliCompress({
+						level: 6
+					})).pipe(fs.createWriteStream(file+".br.tmp")).on("finish", function(){
+						fs.rename(file+".br.tmp", file+".br", function(){
+							debug("<tile> [%s] compressed with brotli", file);
+							done();
+						});
+					});
+				break;
+				case "gz":
+					if (zopfli !== null) {
+						// use zopfli
+						fs.createReadStream(file).pipe(zopfli.createGzip({
+							numiterations: 5, // don't block the queue too long
+						})).pipe(fs.createWriteStream(file+".gz.tmp")).on("finish", function(){
+							fs.rename(file+".gz.tmp", file+".gz", function(){
+								debug("<tile> [%s] compressed with zopfli", file);
+								done();
+							});
+						});
+					} else {
+						// use builtin gzip
+						fs.createReadStream(file).pipe(zlib.createGzip({
+							level: 6
+						})).pipe(fs.createWriteStream(file+".gz.tmp")).on("finish", function(){
+							fs.rename(file+".gz.tmp", file+".gz", function(){
+								debug("<tile> [%s] compressed with gzip", file);
+								done();
+							});
+						});
+					}
+				break;
+				default:
+					done();
+				break;
+			}
+		});
+	});
+};
+
 // stream multiplexer
 tileblaster.prototype._mux = function(fn){
 	var self = this;
@@ -550,7 +656,8 @@ tileblaster.prototype._checktile = function(mapid, z, x, y, r, ext, fn){
 	// check extension
 	if (!!self.maps[mapid].ext && self.maps[mapid].ext.indexOf(ext) < 0) return debug("<check> disallowed extension '%s' for map '%s'", ext, mapid), fn(new Error("Disallowed extension"));
 
-	// FIXME: check retina support?
+	// check density
+	if (!!r && self.maps[mapid].res.indexOf(r) < 0) return debug("<check> disallowed density '%s' for map '%s'", res, mapid), fn(new Error("Disallowed density"));
 
 	// check zoom level
 	var zf = parseFloat(z,10);
@@ -584,4 +691,44 @@ tileblaster.prototype._tilefile = function(mapid, z, x, y, r, e){
 	return (mapid+"/"+z.toFixed(0)+"/"+x.toFixed(0)+"/"+y.toFixed(0)+((!!r)?r:"")+"."+((e) ? e : ""));
 };
 
+// path parsing regular expression
+tileblaster.prototype._pathregx = /\/(([a-z0-9\-\_\.]+)\/((tile\.json)|([0-9]+)\/([0-9]+)\/([0-9]+)(@([0-9]+(\.[0-9]+)?)x)?\.([a-z0-9\.]+)))$/;
+
+// transform path to parameters
+tileblaster.prototype._tilepath = function(p) {
+	var r = (this._pathregx.exec(p));
+	if (!r) return false;
+	if (!r[4]) {
+		return {
+			type: "tile",
+			p: r[1],
+			mapid: r[2],
+			z: parseInt(r[5],10),
+			x: parseInt(r[6],10),
+			y: parseInt(r[7],10),
+			r: r[8],
+			ext: r[11],
+		};
+	} else {
+		return {
+			type: "file",
+			p: r[1],
+			mapid: r[2],
+			file: r[4],
+		};
+	}
+};
+
+// assemble tilejson (good enough)
+tileblaster.prototype._tilejson = function(id, base) {
+	var self = this;
+	return JSON.stringify({
+		tilejson: "2.2.0",
+		minzoom: self.config.maps[id].zoom[0],
+		maxzoom: self.config.maps[id].zoom[1],
+		bounds: self.config.maps[id].bbox,
+		tiles: [ (self.config.base||base)+"/"+id+"/{z}/{x}/{y}"+(self.config.maps[id].res[0]||"")+"."+self.config.maps[id].ext[0] ],
+	});
+};
+
 module.exports = tileblaster;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "tileblaster",
-  "version": "0.2.6",
-  "description": "pretty fast optimizing tile caching proxy",
+  "version": "0.3.2",
+  "description": "pretty fast optimizing & compressing tile caching proxy",
   "main": "lib/tileblaster.js",
   "bin": {
     "tileblaster": "bin/tileblaster.js"
@@ -17,17 +17,19 @@
   },
   "homepage": "https://github.com/yetzt/tileblaster#readme",
   "dependencies": {
-    "debug": "^4",
-    "dur": "^0",
-    "glob": "^7.1",
+    "debug": "^4.3.3",
+    "dur": "^0.0.3",
+    "glob": "^7.2.0",
     "minimist": "^1.2.5",
     "mkdirp": "^1",
+    "node-watch": "^0.7.3",
     "nsa": "^0.2",
-    "quu": "^0.3",
+    "quu": "^0.4.1",
     "request": "^2.88"
   },
   "optionalDependencies": {
     "jpck": "^1.0.2",
+    "node-zopfli": "^2.1.4",
     "pnck": "^1.0.1"
   }
 }

package/readme.md

@@ -28,6 +28,13 @@ upstream upstream_tileblaster {
 server {
 	listen 80;
 	server_name tileblaster;
+	
+	gzip_static on;
+	# brotli_static on; # if ngx_brotli is available
+	
+	if (-f $document_root/$uri.err) {
+		return 204;
+	}
 
 	location / {
 		root /path/to/tileblaster/tiles;
@@ -48,3 +55,4 @@ get the tiles via `http://server/<mapid>/<z>/<x>/<y>[<d>].<ext>`
 * `<z>`, `<x>` and `<z>` are the tile coorinates
 * `<d>` is the optional pixel density marker, for example `@2x`
 * `<ext>` is the extension, for example `png`, `geojson` or `mvt`
+	

package/.gitignore~

@@ -1,4 +0,0 @@
-config.js
-node_modules
-data
-tileblaster.sock

package/renovate.json

@@ -1,5 +0,0 @@
-{
-  "extends": [
-    "config:base"
-  ]
-}