tileblaster 0.4.8 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/UNLICENSE.md +9 -0
- package/bin/tileblaster.js +75 -25
- package/builtins/cache.js +160 -0
- package/builtins/check.js +115 -0
- package/builtins/compress.js +91 -0
- package/builtins/cors.js +33 -0
- package/builtins/deliver.js +34 -0
- package/builtins/dump.js +19 -0
- package/builtins/edit.js +30 -0
- package/builtins/mbtiles.js +52 -0
- package/builtins/modernize.js +57 -0
- package/builtins/noop.js +5 -0
- package/builtins/optimize.js +119 -0
- package/builtins/parse.js +52 -0
- package/builtins/pmtiles.js +80 -0
- package/builtins/sharp.js +46 -0
- package/builtins/tileserver.js +106 -0
- package/builtins/versatiles.js +68 -0
- package/config.dist.js +153 -0
- package/docs/config.md +343 -0
- package/docs/examples.md +257 -0
- package/docs/tileblaster.png +0 -0
- package/docs/tileblaster.svg +261 -0
- package/docs/todo.md +140 -0
- package/lib/debug.js +68 -0
- package/lib/decompress.js +44 -0
- package/lib/load.js +26 -0
- package/lib/mime.js +51 -0
- package/lib/purge.js +109 -0
- package/lib/retrieve.js +15 -0
- package/lib/rfc822date.js +9 -0
- package/lib/router.js +95 -0
- package/lib/store.js +161 -0
- package/lib/strtpl.js +15 -0
- package/lib/tasks.js +50 -0
- package/package.json +25 -13
- package/plugins/example.js +22 -0
- package/readme.md +66 -33
- package/tileblaster.js +445 -0
- package/LICENSE +0 -21
- package/config.js.dist +0 -81
- package/lib/tileblaster.js +0 -787
package/lib/tileblaster.js
DELETED
|
@@ -1,787 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
var fs = require("fs");
|
|
4
|
-
var url = require("url");
|
|
5
|
-
var path = require("path");
|
|
6
|
-
var http = require("http");
|
|
7
|
-
var util = require("util");
|
|
8
|
-
var zlib = require("zlib");
|
|
9
|
-
var stream = require("stream");
|
|
10
|
-
|
|
11
|
-
var phin = require("phin");
|
|
12
|
-
var queue = require("quu");
|
|
13
|
-
var walk = require("klaw");
|
|
14
|
-
var dur = require("dur");
|
|
15
|
-
|
|
16
|
-
// optional dependencies;
|
|
17
|
-
try { var pnck = require("pnck"); } catch (err) { var pnck = null; }
|
|
18
|
-
try { var jpck = require("jpck"); } catch (err) { var jpck = null; }
|
|
19
|
-
try { var zopfli = require("node-zopfli"); } catch (err) { var zopfli = null; }
|
|
20
|
-
try { var versatiles = require("versatiles"); } catch (err) { var versatiles = null; }
|
|
21
|
-
try { var debug = require("debug")("tileblaster"); } catch (err) { var debug = function(){ if (process.env.DEBUG) console.error(...arguments); }}
|
|
22
|
-
|
|
23
|
-
// load package
|
|
24
|
-
var pckg = require("../package.json");
|
|
25
|
-
|
|
26
|
-
function tileblaster(config){
|
|
27
|
-
return (this instanceof tileblaster) ? this.init(config) : new tileblaster(config);
|
|
28
|
-
};
|
|
29
|
-
|
|
30
|
-
// mime types
|
|
31
|
-
tileblaster.prototype.mime = {
|
|
32
|
-
// raster
|
|
33
|
-
png: 'image/png',
|
|
34
|
-
jpg: 'image/jpeg',
|
|
35
|
-
jpeg: 'image/jpeg',
|
|
36
|
-
gif: 'image/gif',
|
|
37
|
-
// vector
|
|
38
|
-
svg: 'image/svg+xml',
|
|
39
|
-
mvt: 'application/vnd.mapbox-vector-tile',
|
|
40
|
-
pbf: 'application/x-protobuf',
|
|
41
|
-
// data
|
|
42
|
-
json: 'text/json',
|
|
43
|
-
geojson: 'text/json',
|
|
44
|
-
topojson: 'text/json',
|
|
45
|
-
// obscure ones:
|
|
46
|
-
arcjson: 'text/json',
|
|
47
|
-
geobson: 'application/octet-stream',
|
|
48
|
-
geoamf: 'application/octet-stream',
|
|
49
|
-
arcamf: 'application/octet-stream',
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
// compression formats
|
|
53
|
-
tileblaster.prototype.comp = [ "gz", "br" ];
|
|
54
|
-
|
|
55
|
-
// initialize
|
|
56
|
-
tileblaster.prototype.init = function(config){
|
|
57
|
-
var self = this;
|
|
58
|
-
|
|
59
|
-
debug("<init> inizializing");
|
|
60
|
-
|
|
61
|
-
// server
|
|
62
|
-
self.srvr = null;
|
|
63
|
-
|
|
64
|
-
// statistics
|
|
65
|
-
self.statistics = {
|
|
66
|
-
hits: 0,
|
|
67
|
-
phits: 0,
|
|
68
|
-
served: 0,
|
|
69
|
-
last: Date.now()
|
|
70
|
-
};
|
|
71
|
-
|
|
72
|
-
// http agents
|
|
73
|
-
self.agents = {};
|
|
74
|
-
|
|
75
|
-
// compression queue
|
|
76
|
-
self.cqueue = queue(1);
|
|
77
|
-
|
|
78
|
-
// have maps with active expires?
|
|
79
|
-
self.expiration = false;
|
|
80
|
-
|
|
81
|
-
// reconfigure
|
|
82
|
-
self.reconfigure(config);
|
|
83
|
-
|
|
84
|
-
return this;
|
|
85
|
-
|
|
86
|
-
};
|
|
87
|
-
|
|
88
|
-
// reconfigure
|
|
89
|
-
tileblaster.prototype.reconfigure = function(config){
|
|
90
|
-
var self = this;
|
|
91
|
-
|
|
92
|
-
debug("<init> reconfiguring");
|
|
93
|
-
|
|
94
|
-
// cache
|
|
95
|
-
self.errcache = {};
|
|
96
|
-
|
|
97
|
-
// keep config
|
|
98
|
-
self.config = config || {};
|
|
99
|
-
|
|
100
|
-
// id
|
|
101
|
-
self.config.id = (!!self.config.id) ? self.config.id : "tileblaster";
|
|
102
|
-
|
|
103
|
-
// socket
|
|
104
|
-
self.config.socket = path.resolve(self.config.socket || "./"+self.config.id+".socket");
|
|
105
|
-
|
|
106
|
-
// tile path
|
|
107
|
-
self.config.tiles = path.resolve(self.config.tiles || "./tiles");
|
|
108
|
-
|
|
109
|
-
// expires (default: never)
|
|
110
|
-
self.config.expires = (dur(self.config.expires) || Infinity);
|
|
111
|
-
|
|
112
|
-
// queue size
|
|
113
|
-
self.config.queue = Math.max(parseInt(self.config.queue,10)||100,1);
|
|
114
|
-
|
|
115
|
-
// extend mime types
|
|
116
|
-
if (!!self.config.mime) Object.keys(self.config.mime).map(function(ext){ self.mime[ext] = self.config.mime[ext] });
|
|
117
|
-
|
|
118
|
-
// queue
|
|
119
|
-
self.queue = queue(self.config.queue);
|
|
120
|
-
|
|
121
|
-
// check configured maps
|
|
122
|
-
self.maps = Object.keys(self.config.maps).reduce(function(maps, id){
|
|
123
|
-
|
|
124
|
-
return maps[id] = (function(map,id){
|
|
125
|
-
|
|
126
|
-
// default zoom
|
|
127
|
-
if (!map.zoom) map.zoom = [0,20];
|
|
128
|
-
|
|
129
|
-
// check for subdomain feature
|
|
130
|
-
if (map.url.indexOf("{s}") >= 0) {
|
|
131
|
-
if (!map.sub) throw new Error("no subdomains configured for map "+id);
|
|
132
|
-
if (typeof map.sub === "string") map.sub = map.sub.split("");
|
|
133
|
-
if (!(map.sub instanceof Array)) throw new Error("invalid 'sub' option for map "+id);
|
|
134
|
-
} else {
|
|
135
|
-
map.sub = false;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
// resolutions
|
|
139
|
-
if (!map.res) map.res = [ "" ];
|
|
140
|
-
if (!(map.res instanceof Array)) map.res = [ map.res ];
|
|
141
|
-
map.res = map.res.filter(function(res){
|
|
142
|
-
return (res === "") || /^@[1-9][0-9]*(\.[0-9]+)?x$/.test(res);
|
|
143
|
-
});
|
|
144
|
-
|
|
145
|
-
// precalculate bbox for zoom levels
|
|
146
|
-
map.bounds = false;
|
|
147
|
-
if (!!map.bbox) {
|
|
148
|
-
|
|
149
|
-
// check if bbox and zoom are valid
|
|
150
|
-
if (!(map.bbox instanceof Array) || map.bbox.length !== 4 || !self._checklnglat([map.bbox[0],map.bbox[1]]) || !self._checklnglat([map.bbox[2],map.bbox[3]])) throw new Error("invalid bounding box: "+JSON.stringify(map.bbox));
|
|
151
|
-
|
|
152
|
-
// sort bbox to ensure wsen
|
|
153
|
-
map.bbox = [
|
|
154
|
-
Math.min(map.bbox[0], map.bbox[2]),
|
|
155
|
-
Math.min(map.bbox[1], map.bbox[3]),
|
|
156
|
-
Math.max(map.bbox[0], map.bbox[2]),
|
|
157
|
-
Math.max(map.bbox[1], map.bbox[3]),
|
|
158
|
-
];
|
|
159
|
-
|
|
160
|
-
map.bounds = [];
|
|
161
|
-
self._range(map.zoom).forEach(function(z){
|
|
162
|
-
map.bounds[z] = {
|
|
163
|
-
"w": self._lngid(map.bbox[0],z),
|
|
164
|
-
"s": self._latid(map.bbox[1],z),
|
|
165
|
-
"e": self._lngid(map.bbox[2],z),
|
|
166
|
-
"n": self._latid(map.bbox[3],z)
|
|
167
|
-
};
|
|
168
|
-
});
|
|
169
|
-
|
|
170
|
-
};
|
|
171
|
-
|
|
172
|
-
// ensure extension list is array
|
|
173
|
-
map.ext = (!!map.ext) ? (map.ext instanceof Array) ? map.ext : [map.ext] : false;
|
|
174
|
-
|
|
175
|
-
// expires
|
|
176
|
-
if (!!map.expires) map.expires = dur(map.expires);
|
|
177
|
-
if (!!map.expires) self.expiration = true;
|
|
178
|
-
if (!map.expires) map.expires = Infinity;
|
|
179
|
-
|
|
180
|
-
// cache
|
|
181
|
-
map.cache = (!map.hasOwnProperty("cache")) ? true : (!!map.cache);
|
|
182
|
-
|
|
183
|
-
// compression
|
|
184
|
-
map.compress = map.compress.filter(function(c){ return (self.comp.indexOf(c) >= 0) });
|
|
185
|
-
|
|
186
|
-
// is versatiles
|
|
187
|
-
map.versatiles = (map.cloudtiles === true || map.versatiles === true);
|
|
188
|
-
|
|
189
|
-
return map;
|
|
190
|
-
|
|
191
|
-
})(self.config.maps[id],id), maps;
|
|
192
|
-
|
|
193
|
-
},{});
|
|
194
|
-
|
|
195
|
-
// build user agent
|
|
196
|
-
if (!self.config.useragent) self.config.useragent = util.format("%s/%s (+%s)", pckg.name, pckg.version, pckg.homepage);
|
|
197
|
-
|
|
198
|
-
// cleanup timer
|
|
199
|
-
self.config.cleanup = (!!self.config.cleanup) ? dur(self.config.cleanup) : false;
|
|
200
|
-
if (!!self.cleaner) clearInterval(self.cleaner);
|
|
201
|
-
if (self.config.cleanup && self.expiration) self.cleaner = setInterval(function(){ self.cleanup(); }, self.config.cleanup).unref(), self.cleanup();
|
|
202
|
-
|
|
203
|
-
};
|
|
204
|
-
|
|
205
|
-
// return server
|
|
206
|
-
tileblaster.prototype.server = function(){
|
|
207
|
-
var self = this;
|
|
208
|
-
|
|
209
|
-
self.srvr = http.createServer(function (req, res) {
|
|
210
|
-
|
|
211
|
-
self.statistics.hits++;
|
|
212
|
-
|
|
213
|
-
// check http method
|
|
214
|
-
if (req.method !== "GET") return debug("<server> invalid method: %s", req.method), (!!self.config.hints&&res.setHeader("x-err-hint","invalid method")), res.statusCode = 405, res.end();
|
|
215
|
-
|
|
216
|
-
// parse request
|
|
217
|
-
var t = self._tilepath(url.parse(req.url).pathname.toLowerCase());
|
|
218
|
-
if (!t) return debug("<server> invalid request: %s", url.parse(req.url).pathname), (!!self.config.hints&&res.setHeader("x-err-hint","invalid request")), res.statusCode = 404, res.end();
|
|
219
|
-
|
|
220
|
-
// check if map exists
|
|
221
|
-
if (!self.maps.hasOwnProperty(t.mapid)) return debug("<server> requested invalid map: %s", t.mapid), (!!self.config.hints&&res.setHeader("x-err-hint","invalid map")), res.statusCode = 404, res.end();
|
|
222
|
-
|
|
223
|
-
// deliver file if requested
|
|
224
|
-
if (t.type === "file") {
|
|
225
|
-
switch (t.file) {
|
|
226
|
-
case "tile.json":
|
|
227
|
-
res.writeHead(200, { "Content-Type": "application/json" });
|
|
228
|
-
res.end(self._tilejson(t.mapid, ((req.headers["x-https"] === "on") ? "https://" : "http://")+req.headers["host"]));
|
|
229
|
-
break;
|
|
230
|
-
default:
|
|
231
|
-
return debug("<server> requested invalid file: %s", t.p), (!!self.config.hints&&res.setHeader("x-err-hint","invalid file")), res.statusCode = 404, res.end();
|
|
232
|
-
break;
|
|
233
|
-
}
|
|
234
|
-
return;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
// res
|
|
238
|
-
t.r = (!!t.r && (self.maps[t.mapid].res.indexOf(t.r) >= 0) ? t.r : false);
|
|
239
|
-
|
|
240
|
-
debug("<server> [%s] requested", t.p);
|
|
241
|
-
|
|
242
|
-
self.tile(t.mapid, t.z, t.x, t.y, t.r, t.ext, function(err, stream, meta){
|
|
243
|
-
if (err) return debug("<server> [%s] error: %s", t.p, err.toString()), (!!self.config.hints&&res.setHeader("x-err-hint",err.toString())), res.statusCode = 204, res.end();
|
|
244
|
-
|
|
245
|
-
// send headers
|
|
246
|
-
res.writeHead(200, { "Content-Type": meta['content-type'] });
|
|
247
|
-
|
|
248
|
-
stream.on("end", function(){
|
|
249
|
-
debug("<server> [%s] done", t.p);
|
|
250
|
-
self.statistics.served++;
|
|
251
|
-
});
|
|
252
|
-
|
|
253
|
-
// pipe stream to http client
|
|
254
|
-
stream.pipe(res);
|
|
255
|
-
|
|
256
|
-
});
|
|
257
|
-
|
|
258
|
-
});
|
|
259
|
-
|
|
260
|
-
// in case self.listen() was called before self.server();
|
|
261
|
-
if (self.listentome) self.listen();
|
|
262
|
-
if (!!self.config.heartbeat) self.heartbeat();
|
|
263
|
-
|
|
264
|
-
return this;
|
|
265
|
-
|
|
266
|
-
};
|
|
267
|
-
|
|
268
|
-
// listen on socket
|
|
269
|
-
tileblaster.prototype.listen = function(){
|
|
270
|
-
var self = this;
|
|
271
|
-
|
|
272
|
-
// wait for server to be ready
|
|
273
|
-
if (!self.srvr) return (self.listentome = true), this;
|
|
274
|
-
|
|
275
|
-
// omit socket if port specified
|
|
276
|
-
if (self.config.port) return self.srvr.listen(self.config.port, function(err){
|
|
277
|
-
if (err) debug("<listen> unable to listen on port %d", self.config.port), process.exit(1);
|
|
278
|
-
debug("<listen> listening on port %d", self.config.port);
|
|
279
|
-
}), self;
|
|
280
|
-
|
|
281
|
-
// listen on socket
|
|
282
|
-
(function(fn){
|
|
283
|
-
(function(next){
|
|
284
|
-
fs.exists(self.config.socket, function(x){
|
|
285
|
-
if (!x) return next();
|
|
286
|
-
fs.unlink(self.config.socket, function(err){
|
|
287
|
-
if (err) return fn(err);
|
|
288
|
-
next();
|
|
289
|
-
});
|
|
290
|
-
});
|
|
291
|
-
})(function(){
|
|
292
|
-
self.srvr.listen(self.config.socket, function(err) {
|
|
293
|
-
if (err) return fn(err);
|
|
294
|
-
fs.chmod(self.config.socket, 0777, fn);
|
|
295
|
-
});
|
|
296
|
-
});
|
|
297
|
-
})(function(err){
|
|
298
|
-
if (err) debug("<listen> unable to listen on socket %s", self.config.socket), process.exit(1);
|
|
299
|
-
debug("<listen> listening on socket %s", self.config.socket);
|
|
300
|
-
});
|
|
301
|
-
|
|
302
|
-
return this;
|
|
303
|
-
|
|
304
|
-
};
|
|
305
|
-
|
|
306
|
-
// start heartbeat server
|
|
307
|
-
tileblaster.prototype.heartbeat = function(){
|
|
308
|
-
var self = this;
|
|
309
|
-
|
|
310
|
-
if (!self.config.heartbeat) return;
|
|
311
|
-
|
|
312
|
-
// check if nsa dependency is installed
|
|
313
|
-
try {
|
|
314
|
-
var nsa = require("nsa");
|
|
315
|
-
} catch (err) {
|
|
316
|
-
return debug("<heartbeat> missing dependency: nsa");
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
self.nsa = nsa({
|
|
320
|
-
server: self.config.heartbeat,
|
|
321
|
-
service: self.config.id,
|
|
322
|
-
interval: "10s"
|
|
323
|
-
}).start(function(){
|
|
324
|
-
// send stats every five minutes
|
|
325
|
-
setInterval(function(){
|
|
326
|
-
|
|
327
|
-
var h = self.statistics.hits - self.statistics.phits;
|
|
328
|
-
var t = Date.now() - self.statistics.last;
|
|
329
|
-
self.statistics.phits = self.statistics.hits;
|
|
330
|
-
self.statistics.last = Date.now();
|
|
331
|
-
var s = {
|
|
332
|
-
"req/s": ((h/t)*1000).toFixed(2),
|
|
333
|
-
"served": self.statistics.served
|
|
334
|
-
};
|
|
335
|
-
self.nsa.send(s);
|
|
336
|
-
debug("<stat> %s req/s, %s served", s["req/s"], s.served);
|
|
337
|
-
},60000).unref();
|
|
338
|
-
|
|
339
|
-
});
|
|
340
|
-
|
|
341
|
-
(function(terminate){
|
|
342
|
-
|
|
343
|
-
process.on("SIGTERM", function(){ terminate("SIGTERM"); });
|
|
344
|
-
process.on("SIGINT", function(){ terminate("SIGINT"); });
|
|
345
|
-
|
|
346
|
-
})(function(signal){
|
|
347
|
-
debug("<terminate> %s", signal);
|
|
348
|
-
debug("<heartbeat> statistics: %j", self.statistics);
|
|
349
|
-
self.nsa.end(function(){
|
|
350
|
-
process.exit(0);
|
|
351
|
-
});
|
|
352
|
-
});
|
|
353
|
-
|
|
354
|
-
return this;
|
|
355
|
-
};
|
|
356
|
-
|
|
357
|
-
// delete expired tiles
|
|
358
|
-
tileblaster.prototype.cleanup = function(){
|
|
359
|
-
var self = this;
|
|
360
|
-
|
|
361
|
-
// clean up error cache
|
|
362
|
-
var d = Date.now();
|
|
363
|
-
self.errcache = Object.entries(self.errcache).reduce(function(errcache, [ key, value ]){
|
|
364
|
-
if (value.until < d) errcache[key] = value;
|
|
365
|
-
return errcache;
|
|
366
|
-
},{});
|
|
367
|
-
|
|
368
|
-
// delete queue
|
|
369
|
-
var deletequeue = queue(1000);
|
|
370
|
-
var now = Date.now();
|
|
371
|
-
|
|
372
|
-
// clean up map tiles
|
|
373
|
-
Object.keys(self.maps).filter(function(mapid){
|
|
374
|
-
return (!!self.maps[mapid].expires);
|
|
375
|
-
}).forEach(function(mapid){
|
|
376
|
-
self.queue.push(function(done){
|
|
377
|
-
debug("<cleanup> start: %s", mapid);
|
|
378
|
-
walk(path.resolve(self.config.tiles, mapid)).on("data", function(f){
|
|
379
|
-
if (f.stats.isFile() && f.stats.mtimeMs+self.maps[mapid].expires < now) deletequeue.push(function(done){
|
|
380
|
-
fs.unlink(f.path, done);
|
|
381
|
-
});
|
|
382
|
-
}).on("error", function(err){
|
|
383
|
-
if (err.code !== "ENOENT") debug("<cleanup> err: %s", err);
|
|
384
|
-
else debug("<cleanup> done: %s", mapid);
|
|
385
|
-
done();
|
|
386
|
-
}).on("end", function(){
|
|
387
|
-
debug("<cleanup> done: %s", mapid);
|
|
388
|
-
done();
|
|
389
|
-
});
|
|
390
|
-
});
|
|
391
|
-
});
|
|
392
|
-
|
|
393
|
-
return this;
|
|
394
|
-
};
|
|
395
|
-
|
|
396
|
-
// get tile
|
|
397
|
-
tileblaster.prototype.tile = function(mapid, z, x, y, r, e, fn){
|
|
398
|
-
var self = this;
|
|
399
|
-
|
|
400
|
-
// optionalize r and e
|
|
401
|
-
if (typeof r === 'function') var e = r, r = false;
|
|
402
|
-
if (typeof e === 'function') var fn = e, e = null;
|
|
403
|
-
|
|
404
|
-
// generate tile filename
|
|
405
|
-
var tilefile = self._tilefile(mapid, z, x, y, r, e);
|
|
406
|
-
|
|
407
|
-
debug("<tile> [%s] requested", tilefile);
|
|
408
|
-
|
|
409
|
-
// check tile
|
|
410
|
-
self._checktile(mapid, z, x, y, r, e, function(err){
|
|
411
|
-
if (err) return fn(err);
|
|
412
|
-
|
|
413
|
-
debug("<tile> [%s] valid", tilefile);
|
|
414
|
-
|
|
415
|
-
// check for cached 404 tiles
|
|
416
|
-
if (!!self.errcache[tilefile] && (self.errcache[tilefile].until > Date.now())) return fn(new Error("Known bad tile: "+self.errcache[tilefile].err), null);
|
|
417
|
-
|
|
418
|
-
// resolve tile path
|
|
419
|
-
var tilepath = path.resolve(self.config.tiles, tilefile);
|
|
420
|
-
|
|
421
|
-
// construct upstream tile url
|
|
422
|
-
var tileurl = self._tileurl(mapid, z, x, y, r, e);
|
|
423
|
-
|
|
424
|
-
(function(next){
|
|
425
|
-
|
|
426
|
-
// versatiles branch
|
|
427
|
-
if (self.maps[mapid].versatiles) return self.versatile(tileurl, mapid, z, x, y, next);
|
|
428
|
-
self.fetchtile(tileurl, mapid, next);
|
|
429
|
-
|
|
430
|
-
})(function(err, tilestream){
|
|
431
|
-
|
|
432
|
-
if (err) {
|
|
433
|
-
if (stream !== null) {
|
|
434
|
-
// cache error tile
|
|
435
|
-
self.errcache[tilefile] = { until: (Date.now()+self.config.maps[mapid].expires), err: err, code: stream };
|
|
436
|
-
|
|
437
|
-
// mark tile as erronous in file system
|
|
438
|
-
fs.writeFile(tilepath+".err", JSON.stringify(self.errcache[tilefile]), function(err){
|
|
439
|
-
if (err) return debug("<tile> [%s.err] error: %s", tilefile, err.toString());
|
|
440
|
-
});
|
|
441
|
-
}
|
|
442
|
-
return debug("<tile> [%s] error: %s", tilefile, err.toString()), fn(err, stream);
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
debug("<tile> [%s] fetched", tilefile);
|
|
446
|
-
|
|
447
|
-
var streams = [];
|
|
448
|
-
|
|
449
|
-
streams.push(function(tilestream){
|
|
450
|
-
// call back with stream
|
|
451
|
-
fn(null, tilestream, {
|
|
452
|
-
'content-type': (self.mime[e])
|
|
453
|
-
});
|
|
454
|
-
});
|
|
455
|
-
|
|
456
|
-
if (!!self.config.maps[mapid].cache) streams.push(function(tilestream){
|
|
457
|
-
|
|
458
|
-
// don't overwrite if tile exists
|
|
459
|
-
fs.access(tilepath, fs.constants.F_OK, function(err){
|
|
460
|
-
if (!err) return debug("<tile> [%s] exists", tilefile);
|
|
461
|
-
|
|
462
|
-
fs.mkdir(path.dirname(tilepath), { recursive: true }, function(err){
|
|
463
|
-
if (err) return debug("<tile> [%s] -- %s", tilefile, err);
|
|
464
|
-
|
|
465
|
-
// save to tmp file, rename when done
|
|
466
|
-
tilestream.pipe(fs.createWriteStream(tilepath+".tmp").on('finish', function(){
|
|
467
|
-
fs.rename(tilepath+".tmp", tilepath, function(){
|
|
468
|
-
|
|
469
|
-
// compress
|
|
470
|
-
if (self.config.maps[mapid].compress instanceof Array && self.config.maps[mapid].compress.length > 0) self.compress(tilepath, self.config.maps[mapid].compress);
|
|
471
|
-
|
|
472
|
-
debug("<tile> [%s] saved", tilefile);
|
|
473
|
-
});
|
|
474
|
-
}));
|
|
475
|
-
|
|
476
|
-
});
|
|
477
|
-
|
|
478
|
-
});
|
|
479
|
-
|
|
480
|
-
});
|
|
481
|
-
|
|
482
|
-
// stream mux
|
|
483
|
-
tilestream.pipe(self.optimize(mapid, e)).pipe(self._mux.apply(self, streams));
|
|
484
|
-
|
|
485
|
-
});
|
|
486
|
-
|
|
487
|
-
});
|
|
488
|
-
|
|
489
|
-
return this;
|
|
490
|
-
};
|
|
491
|
-
|
|
492
|
-
// fetch tile from remote
|
|
493
|
-
tileblaster.prototype.fetchtile = function(tileurl, mapid, fn){
|
|
494
|
-
var self = this;
|
|
495
|
-
|
|
496
|
-
self.queue.push(function(done){
|
|
497
|
-
|
|
498
|
-
// create agent
|
|
499
|
-
var proto = tileurl.substr(0,tileurl.indexOf(":"));
|
|
500
|
-
if (!self.agents.hasOwnProperty(proto)) self.agents[proto] = new require(proto).Agent({ keepAlive: true });
|
|
501
|
-
|
|
502
|
-
debug("<fetchtile> [%s] requested", tileurl);
|
|
503
|
-
var d = 0;
|
|
504
|
-
|
|
505
|
-
phin({
|
|
506
|
-
url: tileurl,
|
|
507
|
-
headers: {
|
|
508
|
-
'user-agent': self.config.useragent, // be nice and tell who we are
|
|
509
|
-
...(self.config.maps[mapid].headers||{}), // extra headers from config
|
|
510
|
-
},
|
|
511
|
-
parse: "none",
|
|
512
|
-
stream: true,
|
|
513
|
-
followRedirects: true,
|
|
514
|
-
compression: true,
|
|
515
|
-
timeout: 10000,
|
|
516
|
-
core: { agent: self.agents[proto] },
|
|
517
|
-
}).then(function(resp){
|
|
518
|
-
|
|
519
|
-
// check mime type, status code, content-length, FIXME: cache!
|
|
520
|
-
if (resp.statusCode !== 200) return resp.stream.destroy(), fn(new Error("status code "+resp.statusCode), resp.statusCode);
|
|
521
|
-
|
|
522
|
-
// parse raw headers if not set
|
|
523
|
-
if (!resp.headers) {
|
|
524
|
-
let rawHeaders = [ ...resp.rawHeaders ];
|
|
525
|
-
resp.headers = {};
|
|
526
|
-
while (rawHeaders.length > 0) resp.headers[ rawHeaders.shift().toLowerCase() ] = rawHeaders.shift();
|
|
527
|
-
};
|
|
528
|
-
|
|
529
|
-
// check headers
|
|
530
|
-
if (!resp.headers['content-type']||(!!self.maps[mapid].mime&&self.maps[mapid].mime.indexOf(resp.headers['content-type'])<0)) return resp.stream.destroy(), fn(new Error("invalid content type "+resp.headers['content-type']), resp.statusCode);
|
|
531
|
-
if (!!resp.headers['content-length']&&parseInt(resp.headers['content-length'],10)===0) return resp.stream.destroy(), fn(new Error("no content"), resp.statusCode);
|
|
532
|
-
|
|
533
|
-
// signal queue when read stream has finished
|
|
534
|
-
resp.stream.once('end', function(){ (!d++)&&done(); });
|
|
535
|
-
|
|
536
|
-
debug("<fetchtile> [%s] received", tileurl);
|
|
537
|
-
|
|
538
|
-
return fn(null, resp.stream, {
|
|
539
|
-
date: (new Date(resp.headers.date||Date.now()).valueOf()),
|
|
540
|
-
size: (parseInt(resp.headers['content-length'],10)||null),
|
|
541
|
-
mime: (resp.headers['content-type']||'application/octet-stream'),
|
|
542
|
-
});
|
|
543
|
-
|
|
544
|
-
}).catch(function(err){
|
|
545
|
-
return debug("<fetchtile> error fetching '%s': %s", tileurl, err), fn(err, null), done();
|
|
546
|
-
});
|
|
547
|
-
|
|
548
|
-
});
|
|
549
|
-
|
|
550
|
-
};
|
|
551
|
-
|
|
552
|
-
// get versatile
|
|
553
|
-
tileblaster.prototype.versatile = function(tileurl, mapid, z, x, y, fn) {
|
|
554
|
-
const self = this;
|
|
555
|
-
if (!versatiles) return fn(new Error("Missing dependency: versatiles"));
|
|
556
|
-
if (!self.maps[mapid].c) self.maps[mapid].c = versatiles(tileurl, { tms: !!self.maps[mapid].tms });
|
|
557
|
-
self.maps[mapid].c.getTile(z,x,y, function(err, buf){
|
|
558
|
-
if (err) return fn(err);
|
|
559
|
-
var strm = new stream.PassThrough;
|
|
560
|
-
strm.write(buf);
|
|
561
|
-
strm.end();
|
|
562
|
-
return fn(null, strm);
|
|
563
|
-
});
|
|
564
|
-
return self;
|
|
565
|
-
};
|
|
566
|
-
|
|
567
|
-
// optimization
|
|
568
|
-
tileblaster.prototype.optimize = function(mapid, ext){
|
|
569
|
-
var self = this;
|
|
570
|
-
|
|
571
|
-
var strm = new stream.PassThrough;
|
|
572
|
-
|
|
573
|
-
if (self.maps[mapid].optimize) switch (ext) {
|
|
574
|
-
case "png":
|
|
575
|
-
if (!pnck) break;
|
|
576
|
-
debug("<optimize> [%s] png", mapid);
|
|
577
|
-
return strm.pipe(pnck(['-o7','-zc8','-zm8','-f5','-quiet','-fix']));
|
|
578
|
-
break;
|
|
579
|
-
case "jpg":
|
|
580
|
-
case "jpeg":
|
|
581
|
-
if (!jpck) break;
|
|
582
|
-
debug("<optimize> [%s] jpg", mapid);
|
|
583
|
-
strm = strm.pipe(jpck({
|
|
584
|
-
optimize: true,
|
|
585
|
-
copy: "none",
|
|
586
|
-
fastcrush: true,
|
|
587
|
-
limit: 102400
|
|
588
|
-
}));
|
|
589
|
-
break;
|
|
590
|
-
}
|
|
591
|
-
|
|
592
|
-
return strm;
|
|
593
|
-
|
|
594
|
-
};
|
|
595
|
-
|
|
596
|
-
// compression
|
|
597
|
-
tileblaster.prototype.compress = function(file, comp){
|
|
598
|
-
var self = this;
|
|
599
|
-
comp.forEach(function(c){
|
|
600
|
-
self.cqueue.push(function(done){
|
|
601
|
-
switch (c) {
|
|
602
|
-
case "br":
|
|
603
|
-
// use builtin brotli
|
|
604
|
-
fs.createReadStream(file).pipe(zlib.createBrotliCompress({
|
|
605
|
-
level: 6
|
|
606
|
-
})).pipe(fs.createWriteStream(file+".br.tmp")).on("finish", function(){
|
|
607
|
-
fs.rename(file+".br.tmp", file+".br", function(){
|
|
608
|
-
debug("<tile> [%s] compressed with brotli", file);
|
|
609
|
-
done();
|
|
610
|
-
});
|
|
611
|
-
});
|
|
612
|
-
break;
|
|
613
|
-
case "gz":
|
|
614
|
-
if (zopfli !== null) {
|
|
615
|
-
// use zopfli
|
|
616
|
-
fs.createReadStream(file).pipe(zopfli.createGzip({
|
|
617
|
-
numiterations: 5, // don't block the queue too long
|
|
618
|
-
})).pipe(fs.createWriteStream(file+".gz.tmp")).on("finish", function(){
|
|
619
|
-
fs.rename(file+".gz.tmp", file+".gz", function(){
|
|
620
|
-
debug("<tile> [%s] compressed with zopfli", file);
|
|
621
|
-
done();
|
|
622
|
-
});
|
|
623
|
-
});
|
|
624
|
-
} else {
|
|
625
|
-
// use builtin gzip
|
|
626
|
-
fs.createReadStream(file).pipe(zlib.createGzip({
|
|
627
|
-
level: 6
|
|
628
|
-
})).pipe(fs.createWriteStream(file+".gz.tmp")).on("finish", function(){
|
|
629
|
-
fs.rename(file+".gz.tmp", file+".gz", function(){
|
|
630
|
-
debug("<tile> [%s] compressed with gzip", file);
|
|
631
|
-
done();
|
|
632
|
-
});
|
|
633
|
-
});
|
|
634
|
-
}
|
|
635
|
-
break;
|
|
636
|
-
default:
|
|
637
|
-
done();
|
|
638
|
-
break;
|
|
639
|
-
}
|
|
640
|
-
});
|
|
641
|
-
});
|
|
642
|
-
};
|
|
643
|
-
|
|
644
|
-
// stream multiplexer
|
|
645
|
-
tileblaster.prototype._mux = function(){
|
|
646
|
-
|
|
647
|
-
// create a passthrough stream for every callback argument
|
|
648
|
-
// call back with created stream
|
|
649
|
-
var streams = Array.from(arguments).map(function(f){
|
|
650
|
-
return (function(s,f){
|
|
651
|
-
return f(s),s;
|
|
652
|
-
})(new stream.PassThrough,f);
|
|
653
|
-
});
|
|
654
|
-
|
|
655
|
-
// multiplex to streams
|
|
656
|
-
return (new stream.Writable({
|
|
657
|
-
write: function(chunk, encoding, done) {
|
|
658
|
-
streams.forEach(function(stream){
|
|
659
|
-
stream.write(chunk, encoding);
|
|
660
|
-
});
|
|
661
|
-
done();
|
|
662
|
-
},
|
|
663
|
-
final: function(done) {
|
|
664
|
-
streams.forEach(function(stream){
|
|
665
|
-
stream.end();
|
|
666
|
-
});
|
|
667
|
-
done();
|
|
668
|
-
}
|
|
669
|
-
}));
|
|
670
|
-
|
|
671
|
-
};
|
|
672
|
-
|
|
673
|
-
// get all integer steps including start and end
|
|
674
|
-
tileblaster.prototype._range = function(z){
|
|
675
|
-
var zooms = [], z = Array.from(z.sort(function(a,b){ return a-b; })); // ensure order and deref
|
|
676
|
-
while (z[0]<=z[1]) zooms.push(z[0]++);
|
|
677
|
-
return zooms;
|
|
678
|
-
};
|
|
679
|
-
|
|
680
|
-
// convert lng to tile x
|
|
681
|
-
tileblaster.prototype._lngid = function(lng,z){
|
|
682
|
-
return (Math.floor((lng+180)/360*Math.pow(2,z)));
|
|
683
|
-
};
|
|
684
|
-
|
|
685
|
-
// convert lat to tile y
|
|
686
|
-
tileblaster.prototype._latid = function(lat,z){
|
|
687
|
-
return (Math.floor((1-Math.log(Math.tan(lat*Math.PI/180) + 1/Math.cos(lat*Math.PI/180))/Math.PI)/2 *Math.pow(2,z)));
|
|
688
|
-
};
|
|
689
|
-
|
|
690
|
-
// check lonlat
|
|
691
|
-
tileblaster.prototype._checklnglat = function(lnglat) {
|
|
692
|
-
if (!(lnglat instanceof Array) || lnglat.length !== 2) return false;
|
|
693
|
-
lnglat = lnglat.map(parseFloat);
|
|
694
|
-
return (!isNaN(lnglat[0]) && !isNaN(lnglat[1]) && lnglat[0] >= -180 && lnglat[0] <= 180 && lnglat[1] >= -90 && lnglat[1] <= 90);
|
|
695
|
-
};
|
|
696
|
-
|
|
697
|
-
// check if a tile meets specifications
|
|
698
|
-
tileblaster.prototype._checktile = function(mapid, z, x, y, r, ext, fn){
|
|
699
|
-
var self = this;
|
|
700
|
-
|
|
701
|
-
// check map identifier
|
|
702
|
-
if (!/^[A-Za-z0-9\-\_]+$/.test(mapid)) return debug("<check> invalid map '%s'", mapid), fn(new Error("Invalid map identifier"));
|
|
703
|
-
if (!self.maps[mapid]) return debug("<check> unknown map '%s'", mapid), fn(new Error("Unknown map identifier"));
|
|
704
|
-
|
|
705
|
-
// check extension
|
|
706
|
-
if (!!self.maps[mapid].ext && self.maps[mapid].ext.indexOf(ext) < 0) return debug("<check> disallowed extension '%s' for map '%s'", ext, mapid), fn(new Error("Disallowed extension"));
|
|
707
|
-
|
|
708
|
-
// check density
|
|
709
|
-
if (!!r && self.maps[mapid].res.indexOf(r) < 0) return debug("<check> disallowed density '%s' for map '%s'", res, mapid), fn(new Error("Disallowed density"));
|
|
710
|
-
|
|
711
|
-
// check zoom level
|
|
712
|
-
var zf = parseFloat(z,10);
|
|
713
|
-
if (zf%1!==0) return debug("<check> invalid zoom float %d", zf), fn(new Error("Disallowed zoom factor"));
|
|
714
|
-
if (zf < self.maps[mapid].zoom[0]) return debug("<check> invalid zoom %d < %d for map '%s'", zf, self.maps[mapid].zoom[0], mapid), fn(new Error("Disallowed zoom factor"));
|
|
715
|
-
if (zf > self.maps[mapid].zoom[1]) return debug("<check> invalid zoom %d > %d for map '%s'", zf, self.maps[mapid].zoom[1], mapid), fn(new Error("Disallowed zoom factor"));
|
|
716
|
-
|
|
717
|
-
// check bbox
|
|
718
|
-
if (!!self.maps[mapid].bbox && (x < self.maps[mapid].bounds[z].w || x > self.maps[mapid].bounds[z].e)) return debug("<check> invalid tile x %d <> [%d-%d@%d] for map '%s'", x, self.maps[mapid].bounds[z].e, self.maps[mapid].bounds[z].w, z, mapid), fn(new Error("Disallowed tile x"));
|
|
719
|
-
if (!!self.maps[mapid].bbox && (y < self.maps[mapid].bounds[z].n || y > self.maps[mapid].bounds[z].s)) return debug("<check> invalid tile y %d <> [%d-%d@%d] for map '%s'", y, self.maps[mapid].bounds[z].n, self.maps[mapid].bounds[z].s, z, mapid), fn(new Error("Disallowed tile y"));
|
|
720
|
-
|
|
721
|
-
fn(null);
|
|
722
|
-
|
|
723
|
-
};
|
|
724
|
-
|
|
725
|
-
// transform parameters to url
|
|
726
|
-
tileblaster.prototype._tileurl = function(mapid, z, x, y, r, e){
|
|
727
|
-
var self = this;
|
|
728
|
-
|
|
729
|
-
// when backend uses tms
|
|
730
|
-
if (!!self.maps[mapid].tms) y = Math.pow(2,z)-y-1;
|
|
731
|
-
|
|
732
|
-
return self.maps[mapid].url
|
|
733
|
-
.replace("{s}", (self.maps[mapid].sub !== false) ? self.maps[mapid].sub[Math.floor(Math.random()*self.maps[mapid].sub.length)] : "")
|
|
734
|
-
.replace("{x}", x.toFixed(0))
|
|
735
|
-
.replace("{y}", y.toFixed(0))
|
|
736
|
-
.replace("{z}", z.toFixed(0))
|
|
737
|
-
.replace("{r}", (!!r) ? r : "")
|
|
738
|
-
.replace("{e}", (e) ? e : "");
|
|
739
|
-
};
|
|
740
|
-
|
|
741
|
-
// transform parameters to filename
|
|
742
|
-
tileblaster.prototype._tilefile = function(mapid, z, x, y, r, e){
|
|
743
|
-
var self = this;
|
|
744
|
-
return (mapid+"/"+z.toFixed(0)+"/"+x.toFixed(0)+"/"+y.toFixed(0)+((!!r)?r:"")+"."+((e) ? e : ""));
|
|
745
|
-
};
|
|
746
|
-
|
|
747
|
-
// path parsing regular expression
|
|
748
|
-
tileblaster.prototype._pathregx = /\/(([a-z0-9\-\_\.]+)\/((tile\.json)|([0-9]+)\/([0-9]+)\/([0-9]+)(@([0-9]+(\.[0-9]+)?)x)?\.([a-z0-9\.]+)))$/;
|
|
749
|
-
|
|
750
|
-
// transform path to parameters
|
|
751
|
-
tileblaster.prototype._tilepath = function(p) {
|
|
752
|
-
var r = (this._pathregx.exec(p));
|
|
753
|
-
if (!r) return false;
|
|
754
|
-
if (!r[4]) {
|
|
755
|
-
return {
|
|
756
|
-
type: "tile",
|
|
757
|
-
p: r[1],
|
|
758
|
-
mapid: r[2],
|
|
759
|
-
z: parseInt(r[5],10),
|
|
760
|
-
x: parseInt(r[6],10),
|
|
761
|
-
y: parseInt(r[7],10),
|
|
762
|
-
r: r[8],
|
|
763
|
-
ext: r[11],
|
|
764
|
-
};
|
|
765
|
-
} else {
|
|
766
|
-
return {
|
|
767
|
-
type: "file",
|
|
768
|
-
p: r[1],
|
|
769
|
-
mapid: r[2],
|
|
770
|
-
file: r[4],
|
|
771
|
-
};
|
|
772
|
-
}
|
|
773
|
-
};
|
|
774
|
-
|
|
775
|
-
// assemble tilejson (good enough)
|
|
776
|
-
tileblaster.prototype._tilejson = function(id, base) {
|
|
777
|
-
var self = this;
|
|
778
|
-
return JSON.stringify({
|
|
779
|
-
tilejson: "2.2.0",
|
|
780
|
-
minzoom: self.config.maps[id].zoom[0],
|
|
781
|
-
maxzoom: self.config.maps[id].zoom[1],
|
|
782
|
-
bounds: self.config.maps[id].bbox,
|
|
783
|
-
tiles: [ (self.config.base||base)+"/"+id+"/{z}/{x}/{y}"+(self.config.maps[id].res[0]||"")+"."+self.config.maps[id].ext[0] ],
|
|
784
|
-
});
|
|
785
|
-
};
|
|
786
|
-
|
|
787
|
-
module.exports = tileblaster;
|