splatone 0.0.21 → 0.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.API_KEY.Flickr +1 -0
- package/README.md +91 -136
- package/browse.js +8 -0
- package/color.js +360 -4
- package/crawler.js +448 -131
- package/package.json +3 -2
- package/plugins/flickr/worker.js +6 -27
- package/public/out/.gitkeep +0 -0
- package/public/out/result.nzzxvl24mi420u0v.json +1 -0
- package/public/out/voronoi/.gitkeep +0 -0
- package/publication/README.md +18 -0
- package/publication/main.tex +85 -0
- package/publication/references.bib +6 -0
- package/views/index.ejs +686 -343
- package/.vscode/mcp.json +0 -12
- package/.vscode/settings.json +0 -7
- package/assets/icon_data_export.png +0 -0
- package/assets/icon_image_download.png +0 -0
- package/assets/screenshot_florida_hex_majorityr.png +0 -0
- package/assets/screenshot_massive_points_bulky.png +0 -0
- package/assets/screenshot_pie_tokyo.png +0 -0
- package/assets/screenshot_sea-mountain_bulky.png +0 -0
- package/assets/screenshot_venice_heat.png +0 -0
- package/assets/screenshot_venice_marker-cluster.png +0 -0
- package/assets/screenshot_venice_simple.png +0 -0
- package/assets/screenshot_voronoi_tokyo.png +0 -0
package/crawler.js
CHANGED
|
@@ -39,14 +39,89 @@ const VIZ_BASE = resolve(__dirname, "visualizer");
|
|
|
39
39
|
const app = express();
|
|
40
40
|
const port = 3000;
|
|
41
41
|
const title = 'Splatone - Multi-Layer Composite Heatmap Viewer';
|
|
42
|
+
const CLI_BASE_COMMAND = process.env.SPLATONE_CLI_BASE ?? 'npx -y -p splatone@latest crawler';
|
|
42
43
|
let pluginsOptions = {};
|
|
43
44
|
let visOptions = {};
|
|
44
45
|
|
|
45
46
|
const flickrLimiter = new Bottleneck({
|
|
46
47
|
maxConcurrent: 6,
|
|
47
|
-
minTime: 700,
|
|
48
|
+
minTime: 700,
|
|
48
49
|
});
|
|
49
50
|
|
|
51
|
+
const VALID_UI_UNITS = new Set(['kilometers', 'meters', 'miles']);
|
|
52
|
+
|
|
53
|
+
function normalizeUiCellSize(value) {
|
|
54
|
+
const num = Number(value);
|
|
55
|
+
if (!Number.isFinite(num) || num < 0) {
|
|
56
|
+
return 0;
|
|
57
|
+
}
|
|
58
|
+
return num;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function parseUiBbox(value) {
|
|
62
|
+
if (!value) return null;
|
|
63
|
+
const parts = String(value).split(',').map(v => Number(v.trim()));
|
|
64
|
+
if (parts.length !== 4 || parts.some(part => !Number.isFinite(part))) {
|
|
65
|
+
throw new Error('--ui-bbox must be "minLon,minLat,maxLon,maxLat"');
|
|
66
|
+
}
|
|
67
|
+
const [minLon, minLat, maxLon, maxLat] = parts;
|
|
68
|
+
if (minLon >= maxLon || minLat >= maxLat) {
|
|
69
|
+
throw new Error('--ui-bbox requires min < max for both lon and lat');
|
|
70
|
+
}
|
|
71
|
+
return [minLon, minLat, maxLon, maxLat];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function extractPolygonFeature(input) {
|
|
75
|
+
if (!input) return null;
|
|
76
|
+
let parsed;
|
|
77
|
+
try {
|
|
78
|
+
parsed = JSON.parse(input);
|
|
79
|
+
} catch (err) {
|
|
80
|
+
throw new Error(`--ui-polygon must be valid GeoJSON: ${err.message}`);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const toFeature = (geometry, properties = {}) => ({
|
|
84
|
+
type: 'Feature',
|
|
85
|
+
properties,
|
|
86
|
+
geometry
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
if (parsed?.type === 'FeatureCollection') {
|
|
90
|
+
const target = parsed.features?.find(f => ['Polygon', 'MultiPolygon'].includes(f?.geometry?.type));
|
|
91
|
+
if (!target) {
|
|
92
|
+
throw new Error('--ui-polygon FeatureCollection must include at least one Polygon or MultiPolygon');
|
|
93
|
+
}
|
|
94
|
+
return toFeature(target.geometry, target.properties ?? {});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if (parsed?.type === 'Feature') {
|
|
98
|
+
if (!parsed.geometry || !['Polygon', 'MultiPolygon'].includes(parsed.geometry.type)) {
|
|
99
|
+
throw new Error('--ui-polygon Feature must contain Polygon or MultiPolygon geometry');
|
|
100
|
+
}
|
|
101
|
+
return toFeature(parsed.geometry, parsed.properties ?? {});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (parsed?.type === 'Polygon' || parsed?.type === 'MultiPolygon') {
|
|
105
|
+
return toFeature(parsed, {});
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
throw new Error('--ui-polygon must be a Polygon/MultiPolygon geometry, Feature, or FeatureCollection');
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function buildUiDefaults(argv) {
|
|
112
|
+
const cellSize = normalizeUiCellSize(argv['ui-cell-size']);
|
|
113
|
+
const unitsInput = argv['ui-units'];
|
|
114
|
+
const units = VALID_UI_UNITS.has(unitsInput) ? unitsInput : 'kilometers';
|
|
115
|
+
const bbox = parseUiBbox(argv['ui-bbox']);
|
|
116
|
+
const polygon = argv['ui-polygon'] ? extractPolygonFeature(argv['ui-polygon']) : null;
|
|
117
|
+
return {
|
|
118
|
+
cellSize,
|
|
119
|
+
units,
|
|
120
|
+
bbox,
|
|
121
|
+
polygon
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
50
125
|
try {
|
|
51
126
|
|
|
52
127
|
// Plugin 読み込み
|
|
@@ -61,6 +136,7 @@ try {
|
|
|
61
136
|
api,
|
|
62
137
|
optionsById: {},
|
|
63
138
|
});
|
|
139
|
+
const installedPluginIds = plugins.list();
|
|
64
140
|
|
|
65
141
|
// Visualizer読み込み
|
|
66
142
|
const all_visualizers = {}; // { [name: string]: class }
|
|
@@ -131,8 +207,8 @@ try {
|
|
|
131
207
|
.option('plugin', {
|
|
132
208
|
group: 'Basic Options',
|
|
133
209
|
alias: 'p',
|
|
134
|
-
choices:
|
|
135
|
-
demandOption:
|
|
210
|
+
choices: installedPluginIds,
|
|
211
|
+
demandOption: false,
|
|
136
212
|
describe: '実行するプラグイン',
|
|
137
213
|
type: 'string'
|
|
138
214
|
})
|
|
@@ -165,6 +241,30 @@ try {
|
|
|
165
241
|
type: 'boolean',
|
|
166
242
|
default: false,
|
|
167
243
|
description: 'デバッグ情報出力'
|
|
244
|
+
}).option('ui-cell-size', {
|
|
245
|
+
group: 'UI Defaults',
|
|
246
|
+
type: 'number',
|
|
247
|
+
default: 0,
|
|
248
|
+
description: '起動時にUIへ設定するセルサイズ (0で自動)'
|
|
249
|
+
}).option('ui-units', {
|
|
250
|
+
group: 'UI Defaults',
|
|
251
|
+
type: 'string',
|
|
252
|
+
choices: ['kilometers', 'meters', 'miles'],
|
|
253
|
+
default: 'kilometers',
|
|
254
|
+
description: 'セルサイズの単位 (kilometers/meters/miles)'
|
|
255
|
+
}).option('ui-bbox', {
|
|
256
|
+
group: 'UI Defaults',
|
|
257
|
+
type: 'string',
|
|
258
|
+
description: 'UI初期表示の矩形範囲。"minLon,minLat,maxLon,maxLat" の形式'
|
|
259
|
+
}).option('ui-polygon', {
|
|
260
|
+
group: 'UI Defaults',
|
|
261
|
+
type: 'string',
|
|
262
|
+
description: 'UI初期表示のポリゴン。Polygon/MultiPolygonを含むGeoJSON文字列'
|
|
263
|
+
}).option('browse-mode', {
|
|
264
|
+
group: 'Basic Options',
|
|
265
|
+
type: 'boolean',
|
|
266
|
+
default: false,
|
|
267
|
+
description: 'ブラウズ専用モード(範囲描画とクロールを無効化)'
|
|
168
268
|
})
|
|
169
269
|
.version()
|
|
170
270
|
.coerce({
|
|
@@ -175,7 +275,7 @@ try {
|
|
|
175
275
|
})()
|
|
176
276
|
*/
|
|
177
277
|
});
|
|
178
|
-
|
|
278
|
+
installedPluginIds.forEach(async (plug) => {
|
|
179
279
|
yargv = await plugins.call(plug, "yargv", yargv);
|
|
180
280
|
})
|
|
181
281
|
|
|
@@ -193,30 +293,65 @@ try {
|
|
|
193
293
|
});
|
|
194
294
|
|
|
195
295
|
yargv = yargv.check(async (argv, options) => {
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
296
|
+
const isBrowseMode = argv['browse-mode'] === true;
|
|
297
|
+
const selectedVisualizers = Object.keys(all_visualizers).filter(v => argv["vis-" + v]);
|
|
298
|
+
if (!isBrowseMode) {
|
|
299
|
+
if (!argv.plugin) {
|
|
300
|
+
throw new Error('プラグインの指定がありません。-p で実行するプラグインを指定してください。');
|
|
301
|
+
}
|
|
302
|
+
if (selectedVisualizers.length === 0) {
|
|
303
|
+
throw new Error('可視化ツールの指定がありません。最低一つは指定してください。');
|
|
304
|
+
}
|
|
305
|
+
if (argv.filed && argv.chopped) {
|
|
306
|
+
console.warn("--filedと--choppedが両方指定されています。--filedが優先されます。");
|
|
307
|
+
argv.chopped = false;
|
|
308
|
+
}
|
|
309
|
+
pluginsOptions = buildPluginsOptions(argv, installedPluginIds);
|
|
310
|
+
visOptions = buildVisualizersOptions(argv, Object.keys(visualizers_));
|
|
311
|
+
pluginsOptions[argv.plugin] = await plugins.call(argv.plugin, 'check', pluginsOptions[argv.plugin]);
|
|
312
|
+
} else {
|
|
313
|
+
pluginsOptions = {};
|
|
314
|
+
visOptions = {};
|
|
315
|
+
if (argv.filed || argv.chopped) {
|
|
316
|
+
if (argv.debugVerbose) {
|
|
317
|
+
console.warn('[browse-mode] --filed/--chopped は無効化されます。');
|
|
318
|
+
}
|
|
319
|
+
argv.filed = false;
|
|
320
|
+
argv.chopped = false;
|
|
321
|
+
}
|
|
322
|
+
if (selectedVisualizers.length > 0 && argv.debugVerbose) {
|
|
323
|
+
console.warn('[browse-mode] --vis-* オプションは無視されます。');
|
|
324
|
+
}
|
|
202
325
|
}
|
|
203
|
-
pluginsOptions = buildPluginsOptions(argv, plugins.list());
|
|
204
|
-
visOptions = buildVisualizersOptions(argv, Object.keys(visualizers_));
|
|
205
|
-
//console.log(visOptions);
|
|
206
|
-
pluginsOptions[argv.plugin] = await plugins.call(argv.plugin, 'check', pluginsOptions[argv.plugin]);
|
|
207
326
|
return true;
|
|
208
327
|
});
|
|
209
328
|
|
|
210
329
|
const argv = await yargv.parseAsync();
|
|
330
|
+
const browseMode = Boolean(argv['browse-mode']);
|
|
331
|
+
|
|
332
|
+
let uiDefaults;
|
|
333
|
+
try {
|
|
334
|
+
uiDefaults = buildUiDefaults(argv);
|
|
335
|
+
} catch (err) {
|
|
336
|
+
console.error(err?.message || err);
|
|
337
|
+
process.exit(1);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
const requestedVisualizerNames = Object.keys(visualizers_).filter(v => argv[`vis-${v}`]);
|
|
341
|
+
const effectiveVisualizerNames = browseMode ? Object.keys(visualizers_) : requestedVisualizerNames;
|
|
211
342
|
|
|
212
343
|
const visualizers = {};
|
|
213
|
-
|
|
214
|
-
|
|
344
|
+
if (!browseMode) {
|
|
345
|
+
for (const vis of effectiveVisualizerNames) {
|
|
346
|
+
visualizers[vis] = visualizers_[vis];
|
|
347
|
+
}
|
|
215
348
|
}
|
|
216
349
|
|
|
217
|
-
|
|
350
|
+
if (!browseMode && argv.plugin) {
|
|
351
|
+
await plugins.call(argv.plugin, 'init', pluginsOptions[argv.plugin]);
|
|
352
|
+
}
|
|
218
353
|
if (argv.debugVerbose) {
|
|
219
|
-
console.table([["Visualizer", Object.keys(visualizers)], ["Plugin", argv.plugin]]);
|
|
354
|
+
console.table([["Visualizer", browseMode ? effectiveVisualizerNames : Object.keys(visualizers)], ["Plugin", argv.plugin || '(browse)']]);
|
|
220
355
|
}
|
|
221
356
|
|
|
222
357
|
|
|
@@ -285,6 +420,8 @@ try {
|
|
|
285
420
|
console.warn('[memory] Failed to read usage stats:', err?.message || err);
|
|
286
421
|
}
|
|
287
422
|
}
|
|
423
|
+
|
|
424
|
+
|
|
288
425
|
/**
|
|
289
426
|
* /api/hexgrid
|
|
290
427
|
* クエリ:
|
|
@@ -311,9 +448,18 @@ try {
|
|
|
311
448
|
title: title,
|
|
312
449
|
lat: DEFAULT_CENTER.lat,
|
|
313
450
|
lon: DEFAULT_CENTER.lon,
|
|
314
|
-
defaultCellSize:
|
|
315
|
-
defaultUnits:
|
|
451
|
+
defaultCellSize: uiDefaults.cellSize,
|
|
452
|
+
defaultUnits: uiDefaults.units,
|
|
316
453
|
defaultKeywords: argv.keywords,
|
|
454
|
+
defaultGeometry: {
|
|
455
|
+
bbox: uiDefaults.bbox,
|
|
456
|
+
polygon: uiDefaults.polygon,
|
|
457
|
+
},
|
|
458
|
+
selectedPlugin: browseMode ? null : argv.plugin,
|
|
459
|
+
selectedVisualizers: browseMode ? [] : Object.keys(visualizers),
|
|
460
|
+
cliBaseCommand: CLI_BASE_COMMAND,
|
|
461
|
+
browseMode,
|
|
462
|
+
clientVisualizers: browseMode ? Object.keys(all_visualizers) : Object.keys(visualizers)
|
|
317
463
|
});
|
|
318
464
|
});
|
|
319
465
|
|
|
@@ -335,9 +481,17 @@ try {
|
|
|
335
481
|
//console.log("disconnected:", socket.id);
|
|
336
482
|
});
|
|
337
483
|
|
|
338
|
-
|
|
484
|
+
const clientVisualizerNames = browseMode ? Object.keys(all_visualizers) : Object.keys(visualizers);
|
|
485
|
+
socket.emit("welcome", { socketId: socket.id, sessionId: sessionId, time: Date.now(), visualizers: clientVisualizerNames });
|
|
339
486
|
//クローリング開始
|
|
340
487
|
socket.on("crawling", async (req) => {
|
|
488
|
+
if (browseMode) {
|
|
489
|
+
socket.emit('toast', {
|
|
490
|
+
text: 'browseコマンドではクロールできません',
|
|
491
|
+
class: 'warning'
|
|
492
|
+
});
|
|
493
|
+
return;
|
|
494
|
+
}
|
|
341
495
|
|
|
342
496
|
try {
|
|
343
497
|
if (sessionId !== req.sessionId) {
|
|
@@ -362,6 +516,13 @@ try {
|
|
|
362
516
|
});
|
|
363
517
|
// クロール範囲指定
|
|
364
518
|
socket.on("target", (req) => {
|
|
519
|
+
if (browseMode) {
|
|
520
|
+
socket.emit('toast', {
|
|
521
|
+
text: 'browseコマンドでは範囲を指定できません',
|
|
522
|
+
class: 'warning'
|
|
523
|
+
});
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
365
526
|
try {
|
|
366
527
|
if (sessionId !== req.sessionId) {
|
|
367
528
|
console.warn("invalid sessionId:", req.sessionId);
|
|
@@ -520,7 +681,7 @@ try {
|
|
|
520
681
|
}
|
|
521
682
|
|
|
522
683
|
// 交差隣接(共有辺で、かつ他Hexの三角形)を付与
|
|
523
|
-
|
|
684
|
+
const triIndex = new Map(triFeatures.map(t => [t.properties.triangleId, t]));
|
|
524
685
|
for (const list of edgeToTriangles.values()) {
|
|
525
686
|
if (list.length < 2) continue; // 共有していなければ隣接なし
|
|
526
687
|
// 同じ辺を共有する全三角形同士で、異なるHexのものを相互に登録
|
|
@@ -554,7 +715,7 @@ try {
|
|
|
554
715
|
}
|
|
555
716
|
crawlers[sessionId] = {};
|
|
556
717
|
processing[sessionId] = 0;
|
|
557
|
-
|
|
718
|
+
targets[sessionId] = { sessionId, hex: hexFC, triangles: trianglesFC, categories, splatonePalette };
|
|
558
719
|
socket.emit("hexgrid", { hex: hexFC, triangles: trianglesFC });
|
|
559
720
|
} catch (e) {
|
|
560
721
|
console.error(e);
|
|
@@ -584,6 +745,7 @@ try {
|
|
|
584
745
|
|
|
585
746
|
const statsItems = (crawler, target) => {
|
|
586
747
|
const progress = [];
|
|
748
|
+
|
|
587
749
|
const categoryCount = Object.keys(target?.categories ?? {}).length;
|
|
588
750
|
const hexCount = target?.hex?.features?.length ?? 0;
|
|
589
751
|
let remaining = categoryCount * hexCount;
|
|
@@ -608,6 +770,87 @@ try {
|
|
|
608
770
|
return { progress, finish: remaining === 0 };
|
|
609
771
|
};
|
|
610
772
|
|
|
773
|
+
function sanitizeCliOptions(argvInput) {
|
|
774
|
+
if (!argvInput || typeof argvInput !== 'object') return {};
|
|
775
|
+
const snapshot = {};
|
|
776
|
+
for (const [key, value] of Object.entries(argvInput)) {
|
|
777
|
+
if (key === '_' || key === '$0') continue;
|
|
778
|
+
if (typeof value === 'function') continue;
|
|
779
|
+
snapshot[key] = value;
|
|
780
|
+
}
|
|
781
|
+
return snapshot;
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
function summarizeCrawlerProgress(crawler = {}, target = {}) {
|
|
785
|
+
const summary = {
|
|
786
|
+
totals: {
|
|
787
|
+
hexes: target?.hex?.features?.length ?? 0,
|
|
788
|
+
triangles: target?.triangles?.features?.length ?? 0,
|
|
789
|
+
categories: Object.keys(target?.categories ?? {}).length,
|
|
790
|
+
crawled: 0,
|
|
791
|
+
remaining: 0,
|
|
792
|
+
expected: 0,
|
|
793
|
+
percent: 0
|
|
794
|
+
},
|
|
795
|
+
hexes: {}
|
|
796
|
+
};
|
|
797
|
+
|
|
798
|
+
for (const [hexId, categories] of Object.entries(crawler ?? {})) {
|
|
799
|
+
const hexStats = {
|
|
800
|
+
categories: {},
|
|
801
|
+
crawled: 0,
|
|
802
|
+
remaining: 0,
|
|
803
|
+
expected: 0,
|
|
804
|
+
percent: 0
|
|
805
|
+
};
|
|
806
|
+
for (const [categoryName, info] of Object.entries(categories ?? {})) {
|
|
807
|
+
const crawled = info?.ids instanceof Set
|
|
808
|
+
? info.ids.size
|
|
809
|
+
: Number(info?.crawled) || 0;
|
|
810
|
+
const remaining = Number(info?.remaining) || 0;
|
|
811
|
+
const total = Number.isFinite(info?.total)
|
|
812
|
+
? Number(info.total)
|
|
813
|
+
: crawled + remaining;
|
|
814
|
+
const percent = total === 0 ? 1 : Math.min(1, crawled / Math.max(1, total));
|
|
815
|
+
hexStats.categories[categoryName] = {
|
|
816
|
+
crawled,
|
|
817
|
+
remaining,
|
|
818
|
+
total,
|
|
819
|
+
percent,
|
|
820
|
+
final: info?.final === true
|
|
821
|
+
};
|
|
822
|
+
hexStats.crawled += crawled;
|
|
823
|
+
hexStats.remaining += remaining;
|
|
824
|
+
hexStats.expected += total;
|
|
825
|
+
}
|
|
826
|
+
hexStats.percent = hexStats.expected === 0
|
|
827
|
+
? 1
|
|
828
|
+
: Math.min(1, hexStats.crawled / Math.max(1, hexStats.expected));
|
|
829
|
+
summary.hexes[hexId] = hexStats;
|
|
830
|
+
summary.totals.crawled += hexStats.crawled;
|
|
831
|
+
summary.totals.remaining += hexStats.remaining;
|
|
832
|
+
summary.totals.expected += hexStats.expected;
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
summary.totals.percent = summary.totals.expected === 0
|
|
836
|
+
? 1
|
|
837
|
+
: Math.min(1, summary.totals.crawled / Math.max(1, summary.totals.expected));
|
|
838
|
+
|
|
839
|
+
return summary;
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
function buildResultContext(crawler, target, argvInput, visualizerNames = []) {
|
|
843
|
+
return {
|
|
844
|
+
generatedAt: new Date().toISOString(),
|
|
845
|
+
hexGrid: target?.hex ?? null,
|
|
846
|
+
triangles: target?.triangles ?? null,
|
|
847
|
+
categories: target?.categories ?? {},
|
|
848
|
+
visualizers: visualizerNames,
|
|
849
|
+
cliOptions: sanitizeCliOptions(argvInput),
|
|
850
|
+
stats: summarizeCrawlerProgress(crawler, target)
|
|
851
|
+
};
|
|
852
|
+
}
|
|
853
|
+
|
|
611
854
|
async function runTask_(taskName, data) {
|
|
612
855
|
const { port1, port2 } = new MessageChannel();
|
|
613
856
|
const filename = resolveWorkerFilename(taskName); // ← file URL (href)
|
|
@@ -640,10 +883,25 @@ try {
|
|
|
640
883
|
}
|
|
641
884
|
//console.log(rtn);
|
|
642
885
|
sessionCrawler[rtn.hexId] ??= {};
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
886
|
+
sessionCrawler[rtn.hexId][rtn.category] ??= {};
|
|
887
|
+
sessionCrawler[rtn.hexId][rtn.category].terms ??= {};
|
|
888
|
+
if (!sessionCrawler[rtn.hexId][rtn.category].terms[rtn.TermId]) {
|
|
889
|
+
//一つ上のTermIdを100%に更新。ラベルはPrefixLabelingなのでrtn.TermId.slice(0,-1)となる。
|
|
890
|
+
const prevTermId = rtn.TermId.slice(0, -1);
|
|
891
|
+
if (sessionCrawler[rtn.hexId][rtn.category].terms[prevTermId] && !sessionCrawler[rtn.hexId][rtn.category].terms[prevTermId].final) {
|
|
892
|
+
sessionCrawler[rtn.hexId][rtn.category].terms[prevTermId].final = true;
|
|
893
|
+
sessionCrawler[rtn.hexId][rtn.category].terms[prevTermId].remaining = 0;
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
sessionCrawler[rtn.hexId][rtn.category].terms[rtn.TermId] ??= {};
|
|
897
|
+
sessionCrawler[rtn.hexId][rtn.category].items ??= featureCollection([]);
|
|
898
|
+
sessionCrawler[rtn.hexId][rtn.category].ids ??= new Set();
|
|
899
|
+
|
|
900
|
+
//定数を作って変数名が長くなるのを防ぐ
|
|
901
|
+
const currentHex = sessionCrawler[rtn.hexId];
|
|
902
|
+
const currentHexCategory = currentHex[rtn.category];
|
|
903
|
+
const idSet = currentHexCategory.ids;
|
|
904
|
+
//Setを使って重複除去
|
|
647
905
|
let duplicateCount = 0;
|
|
648
906
|
const uniqueFeatures = [];
|
|
649
907
|
for (const feature of rtn.photos.features) {
|
|
@@ -657,10 +915,56 @@ try {
|
|
|
657
915
|
}
|
|
658
916
|
uniqueFeatures.push(feature);
|
|
659
917
|
}
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
918
|
+
|
|
919
|
+
//進捗更新。TermIdごとにfinal/remainingを管理。
|
|
920
|
+
currentHexCategory.terms[rtn.TermId].remaining = rtn.remaining;
|
|
921
|
+
currentHexCategory.terms[rtn.TermId].final = rtn.final;
|
|
922
|
+
if (rtn.photos.features.length >= 250 && duplicateCount === rtn.photos.features.length) {
|
|
923
|
+
console.error("[ERROR] ALL DUPLICATE");
|
|
924
|
+
}
|
|
925
|
+
const hexCategoryRemaining = Object.values(currentHexCategory.terms).reduce((sum, term) => sum + (term.remaining || 0), 0);
|
|
926
|
+
currentHexCategory.remaining = hexCategoryRemaining;
|
|
927
|
+
currentHexCategory.total = hexCategoryRemaining + idSet.size;
|
|
928
|
+
currentHexCategory.crawled = idSet.size;
|
|
929
|
+
|
|
930
|
+
const hexRemaining =Object.values(currentHexCategory.terms).reduce((sum, term) => sum + (term.remaining || 0), 0);
|
|
931
|
+
const hexProgress ={
|
|
932
|
+
percent: currentHexCategory.total === 0 ? 1 : Math.min(1, currentHexCategory.crawled / Math.max(1, currentHexCategory.total)),
|
|
933
|
+
total: currentHexCategory.total,
|
|
934
|
+
}
|
|
935
|
+
if (argv.debugVerbose) {
|
|
936
|
+
console.log('INFO:', ` ${rtn.hexId} ${rtn.category} ] dup=${duplicateCount}, out=${rtn.outside}, in=${rtn.photos.features.length} || ${currentHexCategory.crawled} / ${currentHexCategory.total}`);
|
|
937
|
+
}
|
|
938
|
+
const uniqueFeatureCollection = featureCollection(uniqueFeatures);
|
|
939
|
+
sessionCrawler[rtn.hexId][rtn.category].items
|
|
940
|
+
= concatFC(sessionCrawler[rtn.hexId][rtn.category].items, uniqueFeatureCollection);
|
|
941
|
+
io.to(currentSessionId).emit('progress', { hexId: rtn.hexId, currentHex });
|
|
942
|
+
if (!rtn.final) {
|
|
943
|
+
// 次回クロール用に更新
|
|
944
|
+
rtn.nextPluginOptions.forEach((nextPluginOptions) => {
|
|
945
|
+
const workerOptionsClone = {
|
|
946
|
+
plugin: workerOptions.plugin,
|
|
947
|
+
hex: workerOptions.hex,
|
|
948
|
+
triangles: workerOptions.triangles,
|
|
949
|
+
bbox: workerOptions.bbox,
|
|
950
|
+
category: workerOptions.category,
|
|
951
|
+
tags: workerOptions.tags,
|
|
952
|
+
pluginOptions: nextPluginOptions,
|
|
953
|
+
sessionId: workerOptions.sessionId
|
|
954
|
+
};
|
|
955
|
+
api.emit('splatone:start', workerOptionsClone);
|
|
956
|
+
});
|
|
957
|
+
//} else if (finish) {
|
|
958
|
+
} else if (processing[currentSessionId] == 0) {
|
|
959
|
+
if (argv.debugVerbose) {
|
|
960
|
+
console.table(progress);
|
|
961
|
+
}
|
|
962
|
+
api.emit('splatone:finish', workerOptions);
|
|
963
|
+
}
|
|
964
|
+
/*
|
|
965
|
+
sessionCrawler[rtn.hexId][rtn.category].terms[rtn.TermId].final = rtn.final;
|
|
966
|
+
sessionCrawler[rtn.hexId][rtn.category].terms[rtn.TermId].remaining = rtn.remaining;
|
|
967
|
+
|
|
664
968
|
|
|
665
969
|
if (rtn.photos.features.length >= 250 && duplicateCount === rtn.photos.features.length) {
|
|
666
970
|
console.error("ALL DUPLICATE");
|
|
@@ -696,6 +1000,7 @@ try {
|
|
|
696
1000
|
}
|
|
697
1001
|
api.emit('splatone:finish', workerOptions);
|
|
698
1002
|
}
|
|
1003
|
+
*/
|
|
699
1004
|
});
|
|
700
1005
|
const rtn = await piscina.run({ debugVerbose: argv.debugVerbose, port: port2, ...data }, { filename, transferList: [port2] });
|
|
701
1006
|
port1.close();
|
|
@@ -710,120 +1015,132 @@ try {
|
|
|
710
1015
|
idleTimeout: 10_000
|
|
711
1016
|
});
|
|
712
1017
|
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
1018
|
+
if (!browseMode) {
|
|
1019
|
+
await subscribe('splatone:start', async workerOptions => {
|
|
1020
|
+
const currentSessionId = workerOptions.sessionId;
|
|
1021
|
+
processing[currentSessionId] = (processing[currentSessionId] ?? 0) + 1;
|
|
1022
|
+
const safeOptions = JSON.parse(JSON.stringify(workerOptions, (_, value) => {
|
|
1023
|
+
if (typeof value === 'function') return undefined;
|
|
1024
|
+
return value;
|
|
1025
|
+
}));
|
|
1026
|
+
runTask(safeOptions.plugin, safeOptions);
|
|
1027
|
+
});
|
|
723
1028
|
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
1029
|
+
await subscribe('splatone:finish', async workerOptions => {
|
|
1030
|
+
const currentSessionId = workerOptions.sessionId;
|
|
1031
|
+
logHeapUsage(`after-crawl session=${currentSessionId}`);
|
|
1032
|
+
const resultId = uniqid();
|
|
1033
|
+
const result = crawlers[currentSessionId];
|
|
1034
|
+
const target = targets[currentSessionId];
|
|
730
1035
|
|
|
731
|
-
|
|
1036
|
+
let geoJson = Object.fromEntries(
|
|
1037
|
+
Object.entries(visualizers).map(([vis, v]) => [vis, v.getFutureCollection(result, target, visOptions[vis])])
|
|
1038
|
+
);
|
|
732
1039
|
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
await io.to(currentSessionId).timeout(120000).emitWithAck('result', {
|
|
1040
|
+
const visualizerNames = Object.keys(visualizers);
|
|
1041
|
+
const resultContext = buildResultContext(result, target, argv, visualizerNames);
|
|
1042
|
+
const palette = target["splatonePalette"];
|
|
1043
|
+
const resultBundle = {
|
|
1044
|
+
version: 1,
|
|
739
1045
|
resultId,
|
|
740
|
-
geoJson,
|
|
741
|
-
palette: target["splatonePalette"],
|
|
742
|
-
visualizers: Object.keys(visualizers),
|
|
743
1046
|
plugin: argv.plugin,
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
1047
|
+
visualizers: visualizerNames,
|
|
1048
|
+
visOptions,
|
|
1049
|
+
palette,
|
|
1050
|
+
context: resultContext,
|
|
1051
|
+
geoJson
|
|
1052
|
+
};
|
|
1053
|
+
const bundleMeta = {
|
|
1054
|
+
version: resultBundle.version,
|
|
1055
|
+
resultId,
|
|
1056
|
+
plugin: resultBundle.plugin,
|
|
1057
|
+
visualizers: resultBundle.visualizers,
|
|
1058
|
+
visOptions: resultBundle.visOptions,
|
|
1059
|
+
palette: resultBundle.palette,
|
|
1060
|
+
context: resultBundle.context
|
|
1061
|
+
};
|
|
1062
|
+
|
|
1063
|
+
//console.log('[splatone:finish]');
|
|
1064
|
+
let deliveryMode = 'inline';
|
|
1065
|
+
try {
|
|
1066
|
+
if (argv.chopped || argv.filed) {
|
|
1067
|
+
throw new RangeError("Invalid string length");
|
|
751
1068
|
}
|
|
752
|
-
io.to(currentSessionId).timeout(
|
|
753
|
-
|
|
754
|
-
|
|
1069
|
+
await io.to(currentSessionId).timeout(120000).emitWithAck('result', {
|
|
1070
|
+
resultId,
|
|
1071
|
+
bundle: resultBundle
|
|
755
1072
|
});
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
const
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
1073
|
+
} catch (e) {
|
|
1074
|
+
if (e instanceof RangeError && /Invalid string length/.test(String(e.message))) {
|
|
1075
|
+
const msg = ((argv.chopped || argv.filed) ? "ユーザの指定により" : "結果サイズが巨大なので") + (argv.chopped ? "断片化送信" : "保存ファイル送信") + "モードでクライアントに送ります";
|
|
1076
|
+
if (argv.debugVerbose) {
|
|
1077
|
+
console.warn("[WARN] " + msg);
|
|
1078
|
+
}
|
|
1079
|
+
io.to(currentSessionId).timeout(5000).emit('toast', {
|
|
1080
|
+
text: msg,
|
|
1081
|
+
class: "warning"
|
|
1082
|
+
});
|
|
1083
|
+
if (argv.chopped) {
|
|
1084
|
+
deliveryMode = 'chunked';
|
|
1085
|
+
//サイズ集計(GeoJSON部分のみカウント)
|
|
1086
|
+
const total_features = ((s = 0, st = [resultBundle.geoJson], v, seen = new WeakSet) => { for (; st.length;)if ((v = st.pop()) && typeof v === 'object' && !seen.has(v)) { seen.add(v); if (Array.isArray(v?.features)) s += v.features.length; for (const k in v) { const x = v[k]; if (x && typeof x === 'object') st.push(x) } } return s })();
|
|
1087
|
+
let current_features = 0
|
|
1088
|
+
await dfsObject(resultBundle, async ({ path, value, kind, type }) => {
|
|
1089
|
+
if (path.length !== 0) {
|
|
1090
|
+
if (kind === "primitive" || kind === "null") {
|
|
1091
|
+
const ackrtn = await io.to(currentSessionId).timeout(120000).emitWithAck('result-chunk', {
|
|
1092
|
+
resultId,
|
|
1093
|
+
path,
|
|
1094
|
+
kind,
|
|
1095
|
+
type,
|
|
1096
|
+
value
|
|
1097
|
+
});
|
|
1098
|
+
} else if (kind === "object") {
|
|
1099
|
+
if (path.at(-2) == "features" && Number.isInteger(path.at(-1))) {
|
|
1100
|
+
current_features++;
|
|
1101
|
+
}
|
|
1102
|
+
const ackrtn = await io.to(currentSessionId).timeout(120000).emitWithAck('result-chunk', {
|
|
1103
|
+
resultId,
|
|
1104
|
+
path,
|
|
1105
|
+
kind,
|
|
1106
|
+
type,
|
|
1107
|
+
progress: { current: current_features, total: total_features }
|
|
1108
|
+
});
|
|
1109
|
+
} else if (kind === "array") {
|
|
1110
|
+
const ackrtn = await io.to(currentSessionId).timeout(120000).emitWithAck('result-chunk', {
|
|
1111
|
+
resultId,
|
|
1112
|
+
path,
|
|
1113
|
+
kind,
|
|
1114
|
+
type
|
|
1115
|
+
});
|
|
776
1116
|
}
|
|
777
|
-
const ackrtn = await io.to(currentSessionId).timeout(120000).emitWithAck('result-chunk', {
|
|
778
|
-
resultId,
|
|
779
|
-
path,
|
|
780
|
-
kind,
|
|
781
|
-
type,
|
|
782
|
-
progress: { current: current_features, total: total_features }
|
|
783
|
-
});
|
|
784
|
-
//console.log("\tACK", ackrtn);
|
|
785
|
-
} else if (kind === "array") {
|
|
786
|
-
//console.log(path.join("."), "=>", `(${kind}:${type})`);
|
|
787
|
-
const ackrtn = await io.to(currentSessionId).timeout(120000).emitWithAck('result-chunk', {
|
|
788
|
-
resultId,
|
|
789
|
-
path,
|
|
790
|
-
kind,
|
|
791
|
-
type
|
|
792
|
-
});
|
|
793
|
-
//console.log("\tACK", ackrtn);
|
|
794
1117
|
}
|
|
795
|
-
}
|
|
796
|
-
|
|
1118
|
+
});
|
|
1119
|
+
} else {
|
|
1120
|
+
deliveryMode = 'file';
|
|
1121
|
+
try {
|
|
1122
|
+
const outPath = await saveGeoJsonObjectAsStream(resultBundle, 'result.' + resultId + '.json');
|
|
1123
|
+
console.log('saved:', outPath);
|
|
1124
|
+
await io.to(currentSessionId).timeout(120000).emitWithAck('result-file', {
|
|
1125
|
+
resultId,
|
|
1126
|
+
});
|
|
1127
|
+
} catch (err) {
|
|
1128
|
+
console.error('failed:', err);
|
|
1129
|
+
process.exitCode = 1;
|
|
797
1130
|
}
|
|
1131
|
+
}
|
|
1132
|
+
await io.to(currentSessionId).timeout(120000).emitWithAck('result', {
|
|
1133
|
+
resultId,
|
|
1134
|
+
bundle: null,
|
|
1135
|
+
meta: bundleMeta
|
|
798
1136
|
});
|
|
799
|
-
//console.log("finish chunks");
|
|
800
1137
|
} else {
|
|
801
|
-
|
|
802
|
-
try {
|
|
803
|
-
const outPath = await saveGeoJsonObjectAsStream(geoJson, 'result.' + resultId + '.json');
|
|
804
|
-
console.log('saved:', outPath);
|
|
805
|
-
const ackrtn = await io.to(currentSessionId).timeout(120000).emitWithAck('result-file', {
|
|
806
|
-
resultId,
|
|
807
|
-
});
|
|
808
|
-
} catch (err) {
|
|
809
|
-
console.error('failed:', err);
|
|
810
|
-
process.exitCode = 1;
|
|
811
|
-
}
|
|
1138
|
+
throw e; // 他の例外はそのまま
|
|
812
1139
|
}
|
|
813
|
-
await io.to(currentSessionId).timeout(120000).emitWithAck('result', {
|
|
814
|
-
resultId,
|
|
815
|
-
geoJson: null, /*geoJsonは送らない*/
|
|
816
|
-
palette: target["splatonePalette"],
|
|
817
|
-
visualizers: Object.keys(visualizers),
|
|
818
|
-
plugin: argv.plugin,
|
|
819
|
-
visOptions
|
|
820
|
-
});
|
|
821
|
-
console.log("[Done]");
|
|
822
|
-
} else {
|
|
823
|
-
throw e; // 他の例外はそのまま
|
|
824
1140
|
}
|
|
825
|
-
|
|
826
|
-
|
|
1141
|
+
console.log(`[Done] resultId=${resultId} mode=${deliveryMode}`);
|
|
1142
|
+
});
|
|
1143
|
+
}
|
|
827
1144
|
|
|
828
1145
|
server.listen(port, async () => {
|
|
829
1146
|
//console.log(`Server running at http://localhost:${port}`);
|