tileblaster 0.4.8 → 1.0.0

package/docs/config.md

@@ -0,0 +1,343 @@
+# tileblaster configuration
+
+tileblaster configuration lives in a javascript (common js) file, that exports an object containing the configuration.
+
+## Example
+
+See [config.dist.js](./config.dist.js)
+
+## Properties
+
+#### `version`
+
+config file format version. Always `1` fro now.
+
+#### `id`
+
+id of the tileblaster instance, in case you want to run more than one; default: `tileblaster`
+
+#### `treads`
+
+number of worker threads in cluster, default: 1
+
+#### `queue`
+
+number of concurrently processed tiles per worker, default: 12
+
+#### `server.url`
+
+public url including subdirectory
+
+#### `server.mount`
+
+mountpoint override, default: pathname of `config.paths.url`
+
+#### `paths.work`
+
+the base directory where files (sockets, logs, plugins, ...) go; default: `~/tileblaster`
+
+#### `paths.data`
+
+the directory in which cached tiles are saved; default: `${config.paths.work}/data`
+
+#### `paths.logs`
+
+the directory in which cached tiles are saved; default: `${config.paths.work}/logs`
+
+#### `paths.plugins`
+
+the directory from which plugins ar loaded; default: `${config.paths.work}/plugins`
+
+#### `paths.sockets`
+
+the directory i nwhich sockets are created; default: `${config.paths.work}/sockets`
+
+### `listen`
+
+an array of server listen configurations. these contain either `port` or `socket`
+
+``` js
+listen: [{
+	port: 8080, // required
+	host: "localhost", // default localhost
+},{
+	socket: "test.socket", // required, absolute path or relative to ${config.paths.socket}
+	mode: 0o660, // change socket mode to this id
+	group: 1000, // change socket group to this is
+}],
+```
+
+### `plugins`
+
+plugins, relative or absolute paths for local files, npm module names otherwise
+
+``` js
+plugins: {
+	resize: "./resize.js", // relative path, starting with ./
+	convert: "/path/to/convert.js", // absolute path
+	optimize: "someplugin", // npm module
+},
+```
+
+### `maps`
+
+each property of this object represents a map and its workflow. the workflow is an array
+of `builtin` and `plugin` directives, that get processed in the order of their definition.
+
+``` js
+{
+	// a map with id "example"
+	example: [{
+		// a builtin directive, processed first
+		builtin: "something",
+		// ...
+	},{
+		// a plugin directive, processed second
+		plugin: "some-plugin",
+		// ...
+	},{
+		// ...
+	}],
+	anoterMap: {
+		// ...
+	}
+}
+```
+
+### Builtins
+
+#### `cors`
+
+Send [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) Headers to the client. Can be limited by Origin.
+You should do this in your reverse proxy though.
+
+``` js
+{
+	builtin: "cors",
+	origins: [ "https://example.org/" ],
+	// origins: [ "*" ], // ← allow anyone
+}
+```
+
+#### `parse`
+
+Parse a client request. You can override this with your own parse function in case
+your client requires the use of a non-standard request uri or uses GET/POST data or whatever.
+
+``` js
+{
+	builtin: "parse", // /z/x/y@r.ext, other formats need plugins; set params and dest
+	parse: function(req, next){ // override parse function, req is the raw request
+		// do things to get parameters from path
+		next(null, {
+			params: { param: "1" }, // deliver parameters
+			dest: "/path/to/tile/{param}/{param}/blub.{e}{c}", // template for destination
+		});
+	},
+}
+```
+
+#### `check`
+
+Limits request dependent on parameters. You can deny access to tiles to certain zoom levels,
+within a bounding box, to specific extensions and densities, or use your own check function.
+
+``` js
+{
+	builtin: "check",
+	zoom: [ 0, 22 ], // min, max
+	bbox: [ -180, -90, 180, 90 ], // west, south, east, north
+	extensions: [ "png", "jpeg" ], // allowed extensions
+	density: [ "", "@2x", "@3x" ], // allowed density markeers
+	check: function(params, fn) { // override check function, params from parse
+		fn(new Error("Check failed")); // deliver error if check failed
+	}
+}
+```
+
+#### `cache`
+
+Local tile cache.
+
+##### Retrieve cached tiles
+
+When no main tile is availabe, try to load it from cache.
+
+``` js
+{
+	builtin: "cache",
+	skipto: "deliver", // skip to this builtin when tile loaded from cache
+}
+```
+
+##### Store tiles
+
+If a tile is set, store it in the local cache.
+
+``` js
+{
+	builtin: "cache",
+	expires: "30d", // duration or seconds or `true` or `false`
+}
+```
+
+* `expires: 300` - Expires after this amount of seconds
+* `expires: "5y 2w 1d 5h 30m 4s"` - Expires after this [duration](https://www.npmjs.com/package/dur#user-content-time-units)
+* `expires: false` - Expires never
+* `expires: true` - Expires instantly
+
+#### `noop`
+
+Does nothing and has no real purpose.
+
+#### `tileserver`
+
+Fetch a tile from a tileserver via http(s).
+
+``` js
+{
+	builtin: "tileserver",
+	url: "https://{s}.tileserver.example/test/{z}/{x}/{y}{r}.{e}",
+	subdomains: [ "a", "b", "x" ], // random chosen and passed in as {s}
+	tms: true, // y coordinate is inverted
+	headers: {}, // additional headers sent to the backend tileserver
+	status: [ 200 ], // expected status code(s)
+	mimetypes: [ "image/png", "image/jpeg" ], // expected mime types
+	mimetype: "image/png", // overwrite mime type from server
+}
+```
+
+#### `versatiles`
+
+Fetch a tile from a local or remote [versatiles](https://versatiles.org) container.
+
+*Pro Tip:* Don't pay MapBox or "Open"MapTiles for limited access to Open Data! You can get completely free Vectortiles in [Shortbread Schema](https://shortbread.geofabrik.de/schema/) at [download.versatiles.org](https://download.versatiles.org/).
+
+``` js
+{
+	// get tile from versatiles container
+	builtin: "versatiles",
+	url: "https://cdn.example/planet.versatiles",
+	// url: "/path/to/planet.versatiles",
+	tms: false, // y coordinate is inverted
+	headers: { // headers sent to versatiles server
+		"X-Tileblaster": "True",
+	},
+}
+```
+
+#### `pmtiles`
+
+Fetch a tile from a pmtiles container.
+
+``` js
+{
+	// get tile from versatiles container
+	builtin: "pmtiles",
+	url: "https://cdn.example/planet.pmtiles",
+	// url: "/path/to/planet.pmtiles",
+}
+```
+#### `mbtiles`
+
+Retrieve a tile from a mbtiles database.
+
+``` js
+{
+	// get tile from versatiles container
+	builtin: "mbtiles",
+	file: "/path/to/planet.mbtiles",
+}
+```
+
+#### `edit`
+
+Edit vectortiles
+
+``` js
+{
+	builtin: "edit",
+	edit: function(layers){
+
+		// remove unused layer
+		layers = layers.filter(function(layer){
+			return (layer.name !== "unused-layer");
+		});
+
+		return layers;
+	},
+}
+```
+
+#### `resize`
+
+**Not yet implemented** FIXME
+
+#### `modernize`
+
+Convert JPEG or PNG raster tiles to WebP or AVIF.
+The resulting tile with the smallest size and a format the client supports becomes the main tile.
+
+``` js
+{
+	builtin: "modernize",
+	webp: {
+		quality: 90,
+		effort: 4,
+	},
+	avif: {
+		quality: 90,
+		effort: 5,
+	},
+}
+```
+
+#### `optimize`
+
+Optimise raster tiles with `mozjpeg` or `optipng`. Only results smaller than the original are kept.
+
+``` js
+{
+	plugin: "optimize",
+	png: { o: 4 }, // true or opts for optipng
+	jpeg: true, // true or opts for mozjpeg
+}
+```
+
+#### `compress`
+
+Compresses all `data.tiles` with brotli and/or gzip. The tile with the smallest size and a compression the client supports becomes the main tile.
+
+``` js
+{
+	builtin: "compress",
+	brotli: 8, // true or <level> or {opts}
+	gzip: true, // true or <level> or {opts}
+}
+```
+
+
+#### `deliver`
+
+Sends `data.tile` to the client. Adds configured response headers.
+
+``` js
+{
+	"builtin": "deliver",
+	"headers": {
+		"X-My-Header": "This is a very good header"
+	}
+}
+```
+
+#### `dump`
+
+Ends all processing and dumps the contents of `data` the console and browser.
+
+Useful for debugging.
+
+``` js
+{
+	"builtin": "dump"
+}
+```

package/docs/examples.md

@@ -0,0 +1,257 @@
+# Examples
+
+## Tileserver
+
+Serve tiles from a tileserver
+
+``` js
+{
+	example: [{
+		// takes a request for /map/0/1/2@2x.png and breaks it down into z=0, x=1, y=2, r=@2x and e=png
+		builtin: "parse",
+		/* but you can override this if you need to
+		parse: function(req, fn){
+			// `req` is the http request object
+			let fragments = req.url.split("/").pop().split(".");
+			// fn is a callback to deliver the pares parameters
+			fn(null, {
+				z: fragments[0], x: fragments[1], y: fragments[2],
+				e: fragments[3], // extension
+				r: "@2x", // density marker
+				d: 2,     // density
+				w: 512,   // tile width
+			});
+		}
+		*/
+	},{
+		// source tileserver
+		builtin: "tileserver",
+		url: "https://{s}.tileserver.example/{z}/{x}/{y}{r}.{e}?access_token=YOUR_OPENDATAGRIFTBOXTILER_ACCESS_TOKEN",
+		// optional:
+		subdomains: [ "a", "b", "c" ], // random chosen and passed in as {s}
+		tms: false, // {y} is inverted
+		headers: { // additional headers sent to tileserver
+			"Origin": "https://totally-legit-website.example/",
+			"User-Agent": "uncoolest-mobile-map-app"
+		},
+		status: [ 200 ], // expected status code(s)
+		mimetypes: [ "image/png", "image/jpeg" ], // expected mime types
+	},{
+		builtin: "deliver"
+	}]
+}
+```
+
+## Versatiles
+
+Serve tiles from a versatiles container
+
+``` js
+{
+	example: [{
+		builtin: "parse",
+	},{
+		builtin: "versatiles",
+		url: "https://cdn.example/planet.versatiles",
+	},{
+		builtin: "deliver"
+	}]
+}
+```
+
+## PMTiles
+
+Serve tiles from a pmtiles container
+
+``` js
+{
+	example: [{
+		builtin: "parse",
+	},{
+		builtin: "pmtiles",
+		url: "https://cdn.example/planet.pmtiles",
+	},{
+		builtin: "deliver"
+	}]
+}
+```
+
+## MBTiles
+
+Serve tiles from an mbtile database
+
+``` js
+{
+	example: [{
+		builtin: "parse",
+	},{
+		builtin: "mbtiles",
+		file: "/path/to/file.mbtiles",
+	},{
+		builtin: "deliver"
+	}]
+}
+```
+
+## Check parameters
+
+Limit access to tiles by zoom level, bbox, extension or density marker with `check` builtin
+
+``` js
+{
+	example: [{
+		builtin: "parse",
+	},{
+		builtin: "check",
+		zoom: [ 0, 10 ], // min, max
+		bbox: [ -180, -85, 180, 85 ], // west, south, east, north
+		extensions: [ "png", "jpeg" ], // allowed extensions
+		density: [ "", "@2x" ], // allowed density markers
+	},{
+		// source tileserver
+		builtin: "tileserver",
+		url: "https://tileserver.example/{z}/{x}/{y}{r}.{e}",
+	},{
+		builtin: "deliver"
+	}]
+}
+```
+
+## Raster tile optimization and conversion
+
+`optimize` will attemt to reduce the raster tile size by applying image optimization, `mozjpeg` for `jpeg` tiles, `optipng` for `png` tiles.
+`modernize` will convert raster tiles to `webp` and `avif` format (using `sharp`) and deliver those to capable clients.
+
+``` js
+{
+	example: [{
+		builtin: "parse",
+	},{
+		// source tileserver
+		builtin: "tileserver",
+		url: "https://tileserver.example/{z}/{x}/{y}.{e}",
+	},{
+		builtin: "optimize",
+		jpeg: {}, // use mozjpeg via wasm for jpeg tiles
+		png: { o: "3", strip: "all" }, // use optipng via wasm for png tiles
+	},{
+		builtin: "modernize",
+		webp: {
+			quality: 85, // good enough quality
+			effort: 3, // don't use much effort
+		},
+		avif: {
+			quality: 100, // 100% quality
+			lossless: true, // lossless only
+			effort: 9, // a lot of effort
+		}
+	},{
+		builtin: "deliver"
+	}]
+}
+```
+
+## Raster tile image manipulation
+
+Use [sharp](https://www.npmjs.com/package/sharp) to manipulate raster tiles.
+Example: convert a raster tile to "dark mode" by inverting the colors and rotating the hue.
+
+``` js
+{
+	example: [{
+		builtin: "parse",
+	},{
+		// source tileserver
+		builtin: "tileserver",
+		url: "https://tileserver.example/{z}/{x}/{y}.png",
+	},{
+		builtin: "sharp", // use builtin sharp
+		negate: {}, // invert colors using sharp.negate()
+		modulate: { hue: 180 }, // rotate hue by 180° using sharp.modulate({ hue: 180 })
+	},{
+		builtin: "deliver"
+	}]
+}
+```
+
+## Edit vector tiles
+
+Use [vtt](https://npmjs.com/package/vtt) to edit vectortiles.
+
+``` js
+{
+	example: [{
+		builtin: "parse",
+	},{
+		builtin: "tileserver",
+		url: "https://tileserver.example/{z}/{x}/{y}.pbf",
+	},{
+		builtin: "edit", // use builtin edit
+		edit: function(layers){
+			// `layers` is ja JSON representation of the layers in a tile
+			// return the edited `layers` data
+			return layers.filter(function(layer){
+				return layer.name !== "pois"; // remove "pois" layer
+			});
+		}
+	},{
+		builtin: "deliver"
+	}]
+}
+```
+
+## Compression
+
+Precompress tiles with brotli and gzip and deliver them to capable clients.
+Use with compressible tiles (vectortiles, json, svg) and not raster tiles.
+
+``` js
+{
+	example: [{
+		builtin: "parse",
+	},{
+		builtin: "tileserver",
+		url: "https://tileserver.example/{z}/{x}/{y}.pbf",
+	},{
+		builtin: "compress",
+		brotli: 6, // brotli compression level 6
+		gzip: 4, // gzip compression level 4
+	},{
+		builtin: "deliver"
+	}]
+}
+```
+
+## Caching
+
+``` js
+{
+	example: [{
+		builtin: "parse",
+	},{
+		builtin: "cache", // before tile retrieved: check if tile is in cache
+		skipto: "deliver", // skip to this builtin/plugin at cache hit
+	},{
+		// get tile from tileserver
+		builtin: "tileserver",
+		url: "https://tileserver.example/{z}/{x}/{y}",
+	},{
+		builtin: "cache", // after tile retrieved: store tile in cache
+		expires: "30d", // keep in cache for 30 days
+	},{
+		builtin: "deliver",
+	}]
+}
+```
+
+## CORS
+
+Allow Cross-origin resource sharing for certain domains
+
+``` js
+{
+	builtin: "cors",
+	origins: [ "https://example.org/" ], // "*" for any domain
+}
+```
+