tileblaster 0.4.1 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/config.js.dist +13 -13
- package/lib/tileblaster.js +137 -121
- package/package.json +4 -5
- package/readme.md +7 -7
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
This is free and unencumbered software released into the public domain.
|
|
2
|
+
|
|
3
|
+
Anyone is free to copy, modify, publish, use, compile, sell, or distribute this
|
|
4
|
+
software, either in source code form or as a compiled binary, for any purpose,
|
|
5
|
+
commercial or non-commercial, and by any means.
|
|
6
|
+
|
|
7
|
+
In jurisdictions that recognize copyright laws, the author or authors of this
|
|
8
|
+
software dedicate any and all copyright interest in the software to the public
|
|
9
|
+
domain. We make this dedication for the benefit of the public at large and to
|
|
10
|
+
the detriment of our heirs and successors. We intend this dedication to be an
|
|
11
|
+
overt act of relinquishment in perpetuity of all present and future rights to
|
|
12
|
+
this software under copyright law.
|
|
13
|
+
|
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
15
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
16
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
17
|
+
AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
|
18
|
+
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
19
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
20
|
+
|
|
21
|
+
For more information, please refer to <https://unlicense.org/>
|
package/config.js.dist
CHANGED
|
@@ -23,23 +23,23 @@ module.exports = {
|
|
|
23
23
|
|
|
24
24
|
// maps
|
|
25
25
|
"maps": {
|
|
26
|
-
|
|
26
|
+
|
|
27
27
|
// map id, accessible via //server/mapid/z/x/y.ext
|
|
28
28
|
"example": {
|
|
29
|
-
|
|
30
|
-
// backend url:
|
|
29
|
+
|
|
30
|
+
// backend url:
|
|
31
31
|
// * {s} subdomains specified in sub
|
|
32
32
|
// * {z}, {x}, {y} tile coordinates
|
|
33
33
|
// * {e} extension
|
|
34
34
|
// * {r} resolution marker
|
|
35
35
|
"url": "https://{s}.tiles.example.com/tiles/{z}/{x}/{y}{r}.{e}",
|
|
36
|
-
|
|
37
|
-
// url points to
|
|
38
|
-
"
|
|
39
|
-
|
|
36
|
+
|
|
37
|
+
// url points to versatiles container (→ https://github.com/versatiles-org/versatiles-spec)
|
|
38
|
+
"versatiles": false,
|
|
39
|
+
|
|
40
40
|
// backend uses tms instead of zxy
|
|
41
41
|
"tms": false,
|
|
42
|
-
|
|
42
|
+
|
|
43
43
|
// possible extensions
|
|
44
44
|
"ext": ["mvt","json","topojson","png","jpg"],
|
|
45
45
|
|
|
@@ -57,23 +57,23 @@ module.exports = {
|
|
|
57
57
|
|
|
58
58
|
// valid resolution markers
|
|
59
59
|
"res": ["@2x","@4x"],
|
|
60
|
-
|
|
60
|
+
|
|
61
61
|
// optimize tiles
|
|
62
62
|
// * .png with `optipng`
|
|
63
63
|
// * .jpg with `mozjpeg`
|
|
64
64
|
"optimize": true,
|
|
65
|
-
|
|
65
|
+
|
|
66
66
|
// compress tiles (only makes sense for non-rastered tiles like json, pbf, mvt)
|
|
67
67
|
"compress": [ "gz", "br" ],
|
|
68
|
-
|
|
68
|
+
|
|
69
69
|
// write tiles to disk
|
|
70
70
|
"cache": true,
|
|
71
71
|
|
|
72
72
|
// minimum time tiles are kept
|
|
73
73
|
"expires": "1d",
|
|
74
|
-
|
|
74
|
+
|
|
75
75
|
},
|
|
76
|
-
|
|
76
|
+
|
|
77
77
|
// more backends here
|
|
78
78
|
|
|
79
79
|
},
|
package/lib/tileblaster.js
CHANGED
|
@@ -9,17 +9,16 @@ var zlib = require("zlib");
|
|
|
9
9
|
var stream = require("stream");
|
|
10
10
|
|
|
11
11
|
var debug = require("debug")("tileblaster");
|
|
12
|
-
var
|
|
13
|
-
var mkdirp = require("mkdirp");
|
|
12
|
+
var phin = require("phin");
|
|
14
13
|
var queue = require("quu");
|
|
15
14
|
var glob = require("glob");
|
|
16
15
|
var dur = require("dur");
|
|
17
16
|
|
|
18
|
-
// optional dependencies; if only
|
|
17
|
+
// optional dependencies; if only
|
|
19
18
|
try { var pnck = require("pnck"); } catch (err) { var pnck = null; }
|
|
20
19
|
try { var jpck = require("jpck"); } catch (err) { var jpck = null; }
|
|
21
20
|
try { var zopfli = require("node-zopfli"); } catch (err) { var zopfli = null; }
|
|
22
|
-
try { var
|
|
21
|
+
try { var versatiles = require("versatiles"); } catch (err) { var versatiles = null; }
|
|
23
22
|
|
|
24
23
|
// load package
|
|
25
24
|
var pckg = require("../package.json");
|
|
@@ -63,16 +62,19 @@ tileblaster.prototype.init = function(config){
|
|
|
63
62
|
self.srvr = null;
|
|
64
63
|
|
|
65
64
|
// statistics
|
|
66
|
-
self.statistics = {
|
|
67
|
-
hits: 0,
|
|
65
|
+
self.statistics = {
|
|
66
|
+
hits: 0,
|
|
68
67
|
phits: 0,
|
|
69
|
-
served: 0,
|
|
70
|
-
last: Date.now()
|
|
68
|
+
served: 0,
|
|
69
|
+
last: Date.now()
|
|
71
70
|
};
|
|
72
71
|
|
|
72
|
+
// http agents
|
|
73
|
+
self.agents = {};
|
|
74
|
+
|
|
73
75
|
// compression queue
|
|
74
76
|
self.cqueue = queue(1);
|
|
75
|
-
|
|
77
|
+
|
|
76
78
|
// have maps with active expires?
|
|
77
79
|
self.expiration = false;
|
|
78
80
|
|
|
@@ -80,7 +82,7 @@ tileblaster.prototype.init = function(config){
|
|
|
80
82
|
self.reconfigure(config);
|
|
81
83
|
|
|
82
84
|
return this;
|
|
83
|
-
|
|
85
|
+
|
|
84
86
|
};
|
|
85
87
|
|
|
86
88
|
// reconfigure
|
|
@@ -88,13 +90,13 @@ tileblaster.prototype.reconfigure = function(config){
|
|
|
88
90
|
var self = this;
|
|
89
91
|
|
|
90
92
|
debug("<init> reconfiguring");
|
|
91
|
-
|
|
93
|
+
|
|
92
94
|
// cache
|
|
93
95
|
self.errcache = {};
|
|
94
|
-
|
|
96
|
+
|
|
95
97
|
// keep config
|
|
96
98
|
self.config = config || {};
|
|
97
|
-
|
|
99
|
+
|
|
98
100
|
// id
|
|
99
101
|
self.config.id = (!!self.config.id) ? self.config.id : "tileblaster";
|
|
100
102
|
|
|
@@ -106,24 +108,24 @@ tileblaster.prototype.reconfigure = function(config){
|
|
|
106
108
|
|
|
107
109
|
// expires (default: never)
|
|
108
110
|
self.config.expires = (dur(self.config.expires) || Infinity);
|
|
109
|
-
|
|
111
|
+
|
|
110
112
|
// queue size
|
|
111
113
|
self.config.queue = Math.max(parseInt(self.config.queue,10)||100,1);
|
|
112
114
|
|
|
113
115
|
// extend mime types
|
|
114
116
|
if (!!self.config.mime) Object.keys(self.config.mime).map(function(ext){ self.mime[ext] = self.config.mime[ext] });
|
|
115
|
-
|
|
117
|
+
|
|
116
118
|
// queue
|
|
117
119
|
self.queue = queue(self.config.queue);
|
|
118
120
|
|
|
119
121
|
// check configured maps
|
|
120
122
|
self.maps = Object.keys(self.config.maps).reduce(function(maps, id){
|
|
121
|
-
|
|
123
|
+
|
|
122
124
|
return maps[id] = (function(map,id){
|
|
123
|
-
|
|
125
|
+
|
|
124
126
|
// default zoom
|
|
125
127
|
if (!map.zoom) map.zoom = [0,20];
|
|
126
|
-
|
|
128
|
+
|
|
127
129
|
// check for subdomain feature
|
|
128
130
|
if (map.url.indexOf("{s}") >= 0) {
|
|
129
131
|
if (!map.sub) throw new Error("no subdomains configured for map "+mapid);
|
|
@@ -132,21 +134,21 @@ tileblaster.prototype.reconfigure = function(config){
|
|
|
132
134
|
} else {
|
|
133
135
|
map.sub = false;
|
|
134
136
|
}
|
|
135
|
-
|
|
137
|
+
|
|
136
138
|
// resolutions
|
|
137
139
|
if (!map.res) map.res = [ "" ];
|
|
138
140
|
if (!(map.res instanceof Array)) map.res = [ map.res ];
|
|
139
141
|
map.res = map.res.filter(function(res){
|
|
140
142
|
return (res === "") || /^@[1-9][0-9]*(\.[0-9]+)?x$/.test(res);
|
|
141
143
|
});
|
|
142
|
-
|
|
144
|
+
|
|
143
145
|
// precalculate bbox for zoom levels
|
|
144
146
|
map.bounds = false;
|
|
145
147
|
if (!!map.bbox) {
|
|
146
148
|
|
|
147
149
|
// check if bbox and zoom are valid
|
|
148
150
|
if (!(map.bbox instanceof Array) || map.bbox.length !== 4 || !self._checklnglat([map.bbox[0],map.bbox[1]]) || !self._checklnglat([map.bbox[2],map.bbox[3]])) throw new Error("invalid bounding box: "+JSON.stringify(map.bbox));
|
|
149
|
-
|
|
151
|
+
|
|
150
152
|
// sort bbox to ensure wsen
|
|
151
153
|
map.bbox = [
|
|
152
154
|
Math.min(map.bbox[0], map.bbox[2]),
|
|
@@ -154,42 +156,42 @@ tileblaster.prototype.reconfigure = function(config){
|
|
|
154
156
|
Math.max(map.bbox[0], map.bbox[2]),
|
|
155
157
|
Math.max(map.bbox[1], map.bbox[3]),
|
|
156
158
|
];
|
|
157
|
-
|
|
159
|
+
|
|
158
160
|
map.bounds = [];
|
|
159
161
|
self._range(map.zoom).forEach(function(z){
|
|
160
|
-
map.bounds[z] = {
|
|
161
|
-
"w": self._lngid(map.bbox[0],z),
|
|
162
|
-
"s": self._latid(map.bbox[1],z),
|
|
163
|
-
"e": self._lngid(map.bbox[2],z),
|
|
164
|
-
"n": self._latid(map.bbox[3],z)
|
|
162
|
+
map.bounds[z] = {
|
|
163
|
+
"w": self._lngid(map.bbox[0],z),
|
|
164
|
+
"s": self._latid(map.bbox[1],z),
|
|
165
|
+
"e": self._lngid(map.bbox[2],z),
|
|
166
|
+
"n": self._latid(map.bbox[3],z)
|
|
165
167
|
};
|
|
166
168
|
});
|
|
167
|
-
|
|
169
|
+
|
|
168
170
|
};
|
|
169
|
-
|
|
171
|
+
|
|
170
172
|
// ensure extension list is array
|
|
171
173
|
map.ext = (!!map.ext) ? (map.ext instanceof Array) ? map.ext : [map.ext] : false;
|
|
172
|
-
|
|
174
|
+
|
|
173
175
|
// expires
|
|
174
176
|
if (!!map.expires) map.expires = dur(map.expires);
|
|
175
177
|
if (!!map.expires) self.expiration = true;
|
|
176
178
|
if (!map.expires) map.expires = Infinity;
|
|
177
|
-
|
|
179
|
+
|
|
178
180
|
// cache
|
|
179
181
|
map.cache = (!map.hasOwnProperty("cache")) ? true : (!!map.cache);
|
|
180
|
-
|
|
182
|
+
|
|
181
183
|
// compression
|
|
182
184
|
map.compress = map.compress.filter(function(c){ return (self.comp.indexOf(c) >= 0) });
|
|
183
|
-
|
|
184
|
-
// is
|
|
185
|
-
map.
|
|
186
|
-
|
|
185
|
+
|
|
186
|
+
// is versatiles
|
|
187
|
+
map.versatiles = (map.cloudtiles === true || map.versatiles === true);
|
|
188
|
+
|
|
187
189
|
return map;
|
|
188
|
-
|
|
190
|
+
|
|
189
191
|
})(self.config.maps[id],id), maps;
|
|
190
|
-
|
|
192
|
+
|
|
191
193
|
},{});
|
|
192
|
-
|
|
194
|
+
|
|
193
195
|
// build user agent
|
|
194
196
|
if (!self.config.useragent) self.config.useragent = util.format("%s/%s (+%s)", pckg.name, pckg.version, pckg.homepage);
|
|
195
197
|
|
|
@@ -197,15 +199,15 @@ tileblaster.prototype.reconfigure = function(config){
|
|
|
197
199
|
self.config.cleanup = (!!self.config.cleanup) ? dur(self.config.cleanup) : false;
|
|
198
200
|
if (!!self.cleaner) clearInterval(self.cleaner);
|
|
199
201
|
if (self.config.cleanup && self.expiration) self.cleaner = setInterval(function(){ self.cleanup(); }, self.config.cleanup).unref();
|
|
200
|
-
|
|
202
|
+
|
|
201
203
|
};
|
|
202
204
|
|
|
203
205
|
// return server
|
|
204
206
|
tileblaster.prototype.server = function(){
|
|
205
207
|
var self = this;
|
|
206
|
-
|
|
208
|
+
|
|
207
209
|
self.srvr = http.createServer(function (req, res) {
|
|
208
|
-
|
|
210
|
+
|
|
209
211
|
self.statistics.hits++;
|
|
210
212
|
|
|
211
213
|
// check http method
|
|
@@ -239,34 +241,34 @@ tileblaster.prototype.server = function(){
|
|
|
239
241
|
|
|
240
242
|
self.tile(t.mapid, t.z, t.x, t.y, t.r, t.ext, function(err, stream, meta){
|
|
241
243
|
if (err) return debug("<server> [%s] error: %s", t.p, err.toString()), (!!self.config.hints&&res.setHeader("x-err-hint",err.toString())), res.statusCode = 204, res.end();
|
|
242
|
-
|
|
244
|
+
|
|
243
245
|
// send headers
|
|
244
246
|
res.writeHead(200, { "Content-Type": meta['content-type'] });
|
|
245
|
-
|
|
247
|
+
|
|
246
248
|
stream.on("end", function(){
|
|
247
249
|
debug("<server> [%s] done", t.p);
|
|
248
250
|
self.statistics.served++;
|
|
249
251
|
});
|
|
250
|
-
|
|
252
|
+
|
|
251
253
|
// pipe stream to http client
|
|
252
254
|
stream.pipe(res);
|
|
253
|
-
|
|
255
|
+
|
|
254
256
|
});
|
|
255
|
-
|
|
257
|
+
|
|
256
258
|
});
|
|
257
|
-
|
|
259
|
+
|
|
258
260
|
// in case self.listen() was called before self.server();
|
|
259
261
|
if (self.listentome) self.listen();
|
|
260
262
|
if (!!self.config.heartbeat) self.heartbeat();
|
|
261
|
-
|
|
263
|
+
|
|
262
264
|
return this;
|
|
263
|
-
|
|
265
|
+
|
|
264
266
|
};
|
|
265
267
|
|
|
266
268
|
// listen on socket
|
|
267
269
|
tileblaster.prototype.listen = function(){
|
|
268
270
|
var self = this;
|
|
269
|
-
|
|
271
|
+
|
|
270
272
|
// wait for server to be ready
|
|
271
273
|
if (!self.srvr) return (self.listentome = true), this;
|
|
272
274
|
|
|
@@ -296,7 +298,7 @@ tileblaster.prototype.listen = function(){
|
|
|
296
298
|
if (err) debug("<listen> unable to listen on socket %s", self.config.socket), process.exit(1);
|
|
297
299
|
debug("<listen> listening on socket %s", self.config.socket);
|
|
298
300
|
});
|
|
299
|
-
|
|
301
|
+
|
|
300
302
|
return this;
|
|
301
303
|
|
|
302
304
|
};
|
|
@@ -304,7 +306,7 @@ tileblaster.prototype.listen = function(){
|
|
|
304
306
|
// start heartbeat server
|
|
305
307
|
tileblaster.prototype.heartbeat = function(){
|
|
306
308
|
var self = this;
|
|
307
|
-
|
|
309
|
+
|
|
308
310
|
if (!!self.config.heartbeat) self.nsa = require("nsa")({
|
|
309
311
|
server: self.config.heartbeat,
|
|
310
312
|
service: self.config.id,
|
|
@@ -312,7 +314,7 @@ tileblaster.prototype.heartbeat = function(){
|
|
|
312
314
|
}).start(function(){
|
|
313
315
|
// send stats every five minutes
|
|
314
316
|
setInterval(function(){
|
|
315
|
-
|
|
317
|
+
|
|
316
318
|
var h = self.statistics.hits - self.statistics.phits;
|
|
317
319
|
var t = Date.now() - self.statistics.last;
|
|
318
320
|
self.statistics.phits = self.statistics.hits;
|
|
@@ -324,14 +326,14 @@ tileblaster.prototype.heartbeat = function(){
|
|
|
324
326
|
self.nsa.send(s);
|
|
325
327
|
debug("<stat> %s req/s, %s served", s["req/s"], s.served);
|
|
326
328
|
},60000).unref();
|
|
327
|
-
|
|
329
|
+
|
|
328
330
|
});
|
|
329
331
|
|
|
330
332
|
(function(terminate){
|
|
331
|
-
|
|
333
|
+
|
|
332
334
|
process.on("SIGTERM", function(){ terminate("SIGTERM"); });
|
|
333
335
|
process.on("SIGINT", function(){ terminate("SIGINT"); });
|
|
334
|
-
|
|
336
|
+
|
|
335
337
|
})(function(signal){
|
|
336
338
|
debug("<terminate> %s", signal);
|
|
337
339
|
debug("<heartbeat> statistics: %j", self.statistics);
|
|
@@ -352,11 +354,11 @@ tileblaster.prototype.cleanup = function(){
|
|
|
352
354
|
self.errcache = self.errcache.filter(function(e){
|
|
353
355
|
return (e.until > d);
|
|
354
356
|
});
|
|
355
|
-
|
|
357
|
+
|
|
356
358
|
// FIXME: extra process
|
|
357
|
-
|
|
359
|
+
|
|
358
360
|
// clean up map tiles
|
|
359
|
-
Object.keys(self.maps).filter(function(mapid){
|
|
361
|
+
Object.keys(self.maps).filter(function(mapid){
|
|
360
362
|
return (!!self.maps[mapid].expires);
|
|
361
363
|
}).forEach(function(mapid){
|
|
362
364
|
self.queue.push(function(done){
|
|
@@ -369,7 +371,7 @@ tileblaster.prototype.cleanup = function(){
|
|
|
369
371
|
var q = queue(5).done(function(n){
|
|
370
372
|
return debug("<cleanup> [%s] checked %d files, deleted %d", mapid, n, d), done();
|
|
371
373
|
});
|
|
372
|
-
|
|
374
|
+
|
|
373
375
|
files.forEach(function(f){
|
|
374
376
|
q.push(function(next){
|
|
375
377
|
fs.stat(f, function(err, stat){
|
|
@@ -387,14 +389,14 @@ tileblaster.prototype.cleanup = function(){
|
|
|
387
389
|
});
|
|
388
390
|
});
|
|
389
391
|
});
|
|
390
|
-
|
|
392
|
+
|
|
391
393
|
return this;
|
|
392
394
|
};
|
|
393
395
|
|
|
394
396
|
// get tile
|
|
395
397
|
tileblaster.prototype.tile = function(mapid, z, x, y, r, e, fn){
|
|
396
398
|
var self = this;
|
|
397
|
-
|
|
399
|
+
|
|
398
400
|
// optionalize r and e
|
|
399
401
|
if (typeof r === 'function') var e = r, r = false;
|
|
400
402
|
if (typeof e === 'function') var fn = e, e = null;
|
|
@@ -403,28 +405,28 @@ tileblaster.prototype.tile = function(mapid, z, x, y, r, e, fn){
|
|
|
403
405
|
var tilefile = self._tilefile(mapid, z, x, y, r, e);
|
|
404
406
|
|
|
405
407
|
debug("<tile> [%s] requested", tilefile);
|
|
406
|
-
|
|
408
|
+
|
|
407
409
|
// check tile
|
|
408
410
|
self._checktile(mapid, z, x, y, r, e, function(err){
|
|
409
411
|
if (err) return fn(err);
|
|
410
412
|
|
|
411
413
|
debug("<tile> [%s] valid", tilefile);
|
|
412
|
-
|
|
414
|
+
|
|
413
415
|
// check for cached 404 tiles
|
|
414
416
|
if (!!self.errcache[tilefile] && (self.errcache[tilefile].until > Date.now())) return fn(new Error("Known bad tile: "+self.errcache[tilefile].err), null);
|
|
415
|
-
|
|
417
|
+
|
|
416
418
|
// resolve tile path
|
|
417
419
|
var tilepath = path.resolve(self.config.tiles, tilefile);
|
|
418
|
-
|
|
420
|
+
|
|
419
421
|
// construct upstream tile url
|
|
420
422
|
var tileurl = self._tileurl(mapid, z, x, y, r, e);
|
|
421
423
|
|
|
422
424
|
(function(next){
|
|
423
|
-
|
|
424
|
-
//
|
|
425
|
-
if (self.maps[mapid].
|
|
425
|
+
|
|
426
|
+
// versatiles branch
|
|
427
|
+
if (self.maps[mapid].versatiles) return self.versatile(tileurl, mapid, z, x, y, next);
|
|
426
428
|
self.fetchtile(tileurl, mapid, next);
|
|
427
|
-
|
|
429
|
+
|
|
428
430
|
})(function(err, tilestream, meta){
|
|
429
431
|
|
|
430
432
|
if (err) {
|
|
@@ -450,28 +452,27 @@ tileblaster.prototype.tile = function(mapid, z, x, y, r, e, fn){
|
|
|
450
452
|
'content-type': (self.mime[e])
|
|
451
453
|
});
|
|
452
454
|
});
|
|
453
|
-
|
|
455
|
+
|
|
454
456
|
if (!!self.config.maps[mapid].cache) streams.push(function(tilestream){
|
|
455
|
-
|
|
457
|
+
|
|
456
458
|
// don't overwrite if tile exists
|
|
457
459
|
fs.access(tilepath, fs.constants.F_OK, function(err){
|
|
458
460
|
if (!err) return debug("<tile> [%s] exists", tilefile);
|
|
459
461
|
|
|
460
|
-
|
|
462
|
+
fs.mkdir(path.dirname(tilepath), { recursive: true }, function(err){
|
|
463
|
+
if (err) return debug("<tile> [%s] -- %s", tilefile, err);
|
|
461
464
|
|
|
462
465
|
// save to tmp file, rename when done
|
|
463
466
|
tilestream.pipe(fs.createWriteStream(tilepath+".tmp").on('finish', function(){
|
|
464
467
|
fs.rename(tilepath+".tmp", tilepath, function(){
|
|
465
|
-
|
|
468
|
+
|
|
466
469
|
// compress
|
|
467
470
|
if (self.config.maps[mapid].compress instanceof Array && self.config.maps[mapid].compress.length > 0) self.compress(tilepath, self.config.maps[mapid].compress);
|
|
468
|
-
|
|
471
|
+
|
|
469
472
|
debug("<tile> [%s] saved", tilefile);
|
|
470
473
|
});
|
|
471
|
-
}));
|
|
474
|
+
}));
|
|
472
475
|
|
|
473
|
-
}).catch(function(err){
|
|
474
|
-
debug("<tile> [%s] -- %s", tilefile, err);
|
|
475
476
|
});
|
|
476
477
|
|
|
477
478
|
});
|
|
@@ -482,9 +483,9 @@ tileblaster.prototype.tile = function(mapid, z, x, y, r, e, fn){
|
|
|
482
483
|
tilestream.pipe(self.optimize(mapid, e)).pipe(self._mux.apply(self, streams));
|
|
483
484
|
|
|
484
485
|
});
|
|
485
|
-
|
|
486
|
+
|
|
486
487
|
});
|
|
487
|
-
|
|
488
|
+
|
|
488
489
|
return this;
|
|
489
490
|
};
|
|
490
491
|
|
|
@@ -494,50 +495,65 @@ tileblaster.prototype.fetchtile = function(tileurl, mapid, fn){
|
|
|
494
495
|
|
|
495
496
|
self.queue.push(function(done){
|
|
496
497
|
|
|
498
|
+
// create agent
|
|
499
|
+
var proto = tileurl.substr(0,tileurl.indexOf(":"));
|
|
500
|
+
if (!self.agents.hasOwnProperty(proto)) self.agents[proto] = new require(proto).Agent({ keepAlive: true });
|
|
501
|
+
|
|
497
502
|
debug("<fetchtile> [%s] requested", tileurl);
|
|
498
503
|
var d = 0;
|
|
499
504
|
|
|
500
|
-
|
|
501
|
-
method: "GET",
|
|
505
|
+
phin({
|
|
502
506
|
url: tileurl,
|
|
503
|
-
|
|
504
|
-
gzip: true, // some tileservers enforce gzip, so better expect it
|
|
505
|
-
headers: {
|
|
507
|
+
headers: {
|
|
506
508
|
'user-agent': self.config.useragent, // be nice and tell who we are
|
|
507
509
|
...(self.config.maps[mapid].headers||{}), // extra headers from config
|
|
508
|
-
|
|
509
|
-
|
|
510
|
+
},
|
|
511
|
+
parse: "none",
|
|
512
|
+
stream: true,
|
|
513
|
+
followRedirects: true,
|
|
514
|
+
compression: true,
|
|
515
|
+
timeout: 10000,
|
|
516
|
+
core: { agent: self.agents[proto] },
|
|
517
|
+
}).then(function(resp){
|
|
510
518
|
|
|
511
519
|
// check mime type, status code, content-length, FIXME: cache!
|
|
512
|
-
if (resp.statusCode !== 200) return fn(new Error("status code "+resp.statusCode), resp.statusCode);
|
|
513
|
-
|
|
514
|
-
|
|
520
|
+
if (resp.statusCode !== 200) return resp.stream.destroy(), fn(new Error("status code "+resp.statusCode), resp.statusCode);
|
|
521
|
+
|
|
522
|
+
// parse raw headers if not set
|
|
523
|
+
if (!resp.headers) {
|
|
524
|
+
let rawHeaders = [ ...resp.rawHeaders ];
|
|
525
|
+
resp.headers = {};
|
|
526
|
+
while (rawHeaders.length > 0) resp.headers[ rawHeaders.shift().toLowerCase() ] = rawHeaders.shift();
|
|
527
|
+
};
|
|
528
|
+
|
|
529
|
+
// check headers
|
|
530
|
+
if (!resp.headers['content-type']||(!!self.maps[mapid].mime&&self.maps[mapid].mime.indexOf(resp.headers['content-type'])<0)) return resp.stream.destroy(), fn(new Error("invalid content type "+resp.headers['content-type']), resp.statusCode);
|
|
531
|
+
if (!!resp.headers['content-length']&&parseInt(resp.headers['content-length'],10)===0) return resp.stream.destroy(), fn(new Error("no content"), resp.statusCode);
|
|
515
532
|
|
|
516
533
|
// signal queue when read stream has finished
|
|
517
|
-
|
|
534
|
+
resp.stream.once('end', function(){ (!d++)&&done(); });
|
|
518
535
|
|
|
519
536
|
debug("<fetchtile> [%s] received", tileurl);
|
|
520
537
|
|
|
521
|
-
return fn(null,
|
|
538
|
+
return fn(null, resp.stream, {
|
|
522
539
|
date: (new Date(resp.headers.date||Date.now()).valueOf()),
|
|
523
540
|
size: (parseInt(resp.headers['content-length'],10)||null),
|
|
524
541
|
mime: (resp.headers['content-type']||'application/octet-stream'),
|
|
525
542
|
});
|
|
526
|
-
|
|
527
|
-
}).
|
|
543
|
+
|
|
544
|
+
}).catch(function(err){
|
|
528
545
|
return debug("<fetchtile> error fetching '%s': %s", tileurl, err), fn(err, null), done();
|
|
529
|
-
}).on("end", function(){
|
|
530
|
-
(!d++)&&done();
|
|
531
546
|
});
|
|
547
|
+
|
|
532
548
|
});
|
|
533
|
-
|
|
549
|
+
|
|
534
550
|
};
|
|
535
551
|
|
|
536
|
-
// get
|
|
537
|
-
tileblaster.prototype.
|
|
552
|
+
// get versatile
|
|
553
|
+
tileblaster.prototype.versatile = function(tileurl, mapid, z, x, y, fn) {
|
|
538
554
|
const self = this;
|
|
539
|
-
if (!
|
|
540
|
-
if (!self.maps[mapid].c) self.maps[mapid].c =
|
|
555
|
+
if (!versatile) return fn(new Error("Missing dependency: versatiles"));
|
|
556
|
+
if (!self.maps[mapid].c) self.maps[mapid].c = versatiles(tileurl, { tms: !!self.maps[mapid].tms });
|
|
541
557
|
self.maps[mapid].c.getTile(z,x,y, function(err, buffer){
|
|
542
558
|
if (err) return fn(err);
|
|
543
559
|
return fn(null, stream.Readable.from(buffer));
|
|
@@ -548,30 +564,30 @@ tileblaster.prototype.cloudtile = function(tileurl, mapid, z, x, y, fn) {
|
|
|
548
564
|
// optimization
|
|
549
565
|
tileblaster.prototype.optimize = function(mapid, ext){
|
|
550
566
|
var self = this;
|
|
551
|
-
|
|
567
|
+
|
|
552
568
|
var strm = new stream.PassThrough;
|
|
553
|
-
|
|
569
|
+
|
|
554
570
|
if (self.maps[mapid].optimize) switch (ext) {
|
|
555
|
-
case "png":
|
|
571
|
+
case "png":
|
|
556
572
|
if (!pnck) break;
|
|
557
573
|
debug("<optimize> [%s] png", mapid);
|
|
558
574
|
return strm.pipe(pnck(['-o7','-zc8','-zm8','-f5','-quiet','-fix']));
|
|
559
575
|
break;
|
|
560
|
-
case "jpg":
|
|
561
|
-
case "jpeg":
|
|
576
|
+
case "jpg":
|
|
577
|
+
case "jpeg":
|
|
562
578
|
if (!jpck) break;
|
|
563
579
|
debug("<optimize> [%s] jpg", mapid);
|
|
564
|
-
strm = strm.pipe(jpck({
|
|
565
|
-
optimize: true,
|
|
566
|
-
copy: "none",
|
|
567
|
-
fastcrush: true,
|
|
568
|
-
limit: 102400
|
|
580
|
+
strm = strm.pipe(jpck({
|
|
581
|
+
optimize: true,
|
|
582
|
+
copy: "none",
|
|
583
|
+
fastcrush: true,
|
|
584
|
+
limit: 102400
|
|
569
585
|
}));
|
|
570
586
|
break;
|
|
571
587
|
}
|
|
572
|
-
|
|
588
|
+
|
|
573
589
|
return strm;
|
|
574
|
-
|
|
590
|
+
|
|
575
591
|
};
|
|
576
592
|
|
|
577
593
|
// compression
|
|
@@ -625,7 +641,7 @@ tileblaster.prototype.compress = function(file, comp){
|
|
|
625
641
|
// stream multiplexer
|
|
626
642
|
tileblaster.prototype._mux = function(fn){
|
|
627
643
|
var self = this;
|
|
628
|
-
|
|
644
|
+
|
|
629
645
|
// create a passthrough stream for every callback argument
|
|
630
646
|
// call back with created stream
|
|
631
647
|
var streams = Array.from(arguments).map(function(f){
|
|
@@ -633,7 +649,7 @@ tileblaster.prototype._mux = function(fn){
|
|
|
633
649
|
return f(s),s;
|
|
634
650
|
})(new stream.PassThrough,f);
|
|
635
651
|
});
|
|
636
|
-
|
|
652
|
+
|
|
637
653
|
// multiplex to streams
|
|
638
654
|
return (new stream.Writable({
|
|
639
655
|
write: function(chunk, encoding, done) {
|
|
@@ -649,12 +665,12 @@ tileblaster.prototype._mux = function(fn){
|
|
|
649
665
|
done();
|
|
650
666
|
}
|
|
651
667
|
}));
|
|
652
|
-
|
|
668
|
+
|
|
653
669
|
};
|
|
654
670
|
|
|
655
671
|
// get all integer steps including start and end
|
|
656
672
|
tileblaster.prototype._range = function(z){
|
|
657
|
-
var zooms = [], z = Array.from(z.sort(function(a,b){ return a-b; })); // ensure order and deref
|
|
673
|
+
var zooms = [], z = Array.from(z.sort(function(a,b){ return a-b; })); // ensure order and deref
|
|
658
674
|
while (z[0]<=z[1]) zooms.push(z[0]++);
|
|
659
675
|
return zooms;
|
|
660
676
|
};
|
|
@@ -699,7 +715,7 @@ tileblaster.prototype._checktile = function(mapid, z, x, y, r, ext, fn){
|
|
|
699
715
|
// check bbox
|
|
700
716
|
if (!!self.maps[mapid].bbox && (x < self.maps[mapid].bounds[z].w || x > self.maps[mapid].bounds[z].e)) return debug("<check> invalid tile x %d <> [%d-%d@%d] for map '%s'", x, self.maps[mapid].bounds[z].e, self.maps[mapid].bounds[z].w, z, mapid), fn(new Error("Disallowed tile x"));
|
|
701
717
|
if (!!self.maps[mapid].bbox && (y < self.maps[mapid].bounds[z].n || y > self.maps[mapid].bounds[z].s)) return debug("<check> invalid tile y %d <> [%d-%d@%d] for map '%s'", y, self.maps[mapid].bounds[z].n, self.maps[mapid].bounds[z].s, z, mapid), fn(new Error("Disallowed tile y"));
|
|
702
|
-
|
|
718
|
+
|
|
703
719
|
fn(null);
|
|
704
720
|
|
|
705
721
|
};
|
|
@@ -707,10 +723,10 @@ tileblaster.prototype._checktile = function(mapid, z, x, y, r, ext, fn){
|
|
|
707
723
|
// transform parameters to url
|
|
708
724
|
tileblaster.prototype._tileurl = function(mapid, z, x, y, r, e){
|
|
709
725
|
var self = this;
|
|
710
|
-
|
|
726
|
+
|
|
711
727
|
// when backend uses tms
|
|
712
728
|
if (!!self.maps[mapid].tms) y = Math.pow(2,z)-y-1;
|
|
713
|
-
|
|
729
|
+
|
|
714
730
|
return self.maps[mapid].url
|
|
715
731
|
.replace("{s}", (self.maps[mapid].sub !== false) ? self.maps[mapid].sub[Math.floor(Math.random()*self.maps[mapid].sub.length)] : "")
|
|
716
732
|
.replace("{x}", x.toFixed(0))
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tileblaster",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.3",
|
|
4
4
|
"description": "pretty fast optimizing & compressing tile caching proxy",
|
|
5
5
|
"main": "lib/tileblaster.js",
|
|
6
6
|
"bin": {
|
|
@@ -21,14 +21,13 @@
|
|
|
21
21
|
"dur": "^0.0.3",
|
|
22
22
|
"glob": "^8.1.0",
|
|
23
23
|
"minimist": "^1.2.7",
|
|
24
|
-
"mkdirp": "^2.0.0",
|
|
25
24
|
"node-watch": "^0.7.3",
|
|
26
25
|
"nsa": "^0.2",
|
|
27
|
-
"
|
|
28
|
-
"
|
|
26
|
+
"phin": "^3.7.0",
|
|
27
|
+
"quu": "^0.4.3"
|
|
29
28
|
},
|
|
30
29
|
"optionalDependencies": {
|
|
31
|
-
"
|
|
30
|
+
"versatiles": "^0.2.0",
|
|
32
31
|
"jpck": "^1.0.2",
|
|
33
32
|
"node-zopfli": "^2.1.4",
|
|
34
33
|
"pnck": "^1.0.1"
|
package/readme.md
CHANGED
|
@@ -6,13 +6,13 @@ tileblaster is a map tile caching (and optimizing) proxy, designed to run with n
|
|
|
6
6
|
|
|
7
7
|
`npm i tileblaster -g`
|
|
8
8
|
|
|
9
|
-
use `--no-optional` if you don't want tile optimization or [
|
|
9
|
+
use `--no-optional` if you don't want tile optimization or [versatiles](https://github.com/versatiles-org/versatiles-spec) support.
|
|
10
10
|
|
|
11
11
|
## run
|
|
12
12
|
|
|
13
13
|
`tileblaster /path/to/config.js`
|
|
14
14
|
|
|
15
|
-
Use [
|
|
15
|
+
Use [pm2](https://npmjs.com/package/pm2), [nodemon](https://npmjs.com/package/nodemon), [forever](https://npmjs.com/package/forever) or similar to run tileblaster as service;
|
|
16
16
|
|
|
17
17
|
## configuration
|
|
18
18
|
|
|
@@ -28,10 +28,10 @@ upstream upstream_tileblaster {
|
|
|
28
28
|
server {
|
|
29
29
|
listen 80;
|
|
30
30
|
server_name tileblaster;
|
|
31
|
-
|
|
31
|
+
|
|
32
32
|
gzip_static on;
|
|
33
33
|
# brotli_static on; # if ngx_brotli is available
|
|
34
|
-
|
|
34
|
+
|
|
35
35
|
if (-f $document_root/$uri.err) {
|
|
36
36
|
return 204;
|
|
37
37
|
}
|
|
@@ -50,9 +50,9 @@ server {
|
|
|
50
50
|
## usage
|
|
51
51
|
|
|
52
52
|
get the tiles via `http://server/<mapid>/<z>/<x>/<y>[<d>].<ext>`
|
|
53
|
-
|
|
53
|
+
|
|
54
54
|
* `<mapid>` is the map id specified in your `config.js`
|
|
55
55
|
* `<z>`, `<x>` and `<z>` are the tile coorinates
|
|
56
56
|
* `<d>` is the optional pixel density marker, for example `@2x`
|
|
57
|
-
* `<ext>` is the extension, for example `png`, `geojson` or `
|
|
58
|
-
|
|
57
|
+
* `<ext>` is the extension, for example `png`, `geojson` or `pbf`
|
|
58
|
+
|