splatone 0.0.3 → 0.0.5

package/README.md

@@ -13,8 +13,14 @@ SNSのジオタグ付きポストをキーワードに基づいて収集する
 
 ## 既知のバグ
 
-- JSON.stringify(json)で変換できる大きさに制限があり、数十万件等の大きな結果を生み出すクエリは、クロール後、結果のブラウザへの転送で失敗します。
-  
+- <s>JSON.stringify(json)で変換できる大きさに制限があり、数十万件等の大きな結果を生み出すクエリは、クロール後、結果のブラウザへの転送で失敗します。</s>
+- サイズの問題は一部解決しました。ただし、**データの保存(ダウンロード)はまだできません。**
+  - 問題はサーバからブラウザへ結果を渡す時にJSON.stringify(obj)で結果データをJSON文字列にする時にヒープを食いつぶしています。 とりあえず可視化だけ出来るように、データが大きいと自動で分割して送信するようにしています。ただし、送受信に時間がかかります。（左下のプログレスバーに進捗が表示されます）
+  - **--chopped**オプションをつけると強制的に分割送信します。(ただ遅くなるだけなので意味ないです)
+  - データ保存(ダウンロード)する際にブラウザ上でJSON.stringify(json)を呼ぶ必要があり、そこで詰まります。これの解決策は別途実装します。
+
+![](/assets/screenshot_massive_points_bulky.png)
+
 # 使い方
 
 - [Node.js](https://nodejs.org/ja/download)をインストール後、npxで実行します。
@@ -27,11 +33,15 @@ $ npx -y -- splatone@latest crawler --help
 使い方: crawler.js [options]
 
 Basic Options
-  -p, --plugin    実行するプラグイン
-                                   [文字列] [必須] [選択してください: "flickr"]
+  -p, --plugin    実行するプラグイン[文字列] [必須] [選択してください: "flickr"]
   -o, --options   プラグインオプション               [文字列] [デフォルト: "{}"]
   -k, --keywords  検索キーワード(|区切り)                  [文字列] [デフォルト:
                        "nature,tree,flower|building,house|water,sea,river,pond"]
+  -c, --chopped   大きなデータを細分化して送信する
+                                                       [真偽] [デフォルト: true]
+
+Debug
+      --debug-verbose  デバッグ情報出力               [真偽] [デフォルト: false]
 
 Visualization (最低一つの指定が必須です)
       --vis-bulky           全データをCircleMarkerとして地図上に表示
@@ -41,7 +51,7 @@ Visualization (最低一つの指定が必須です)
 
 オプション:
       --help     ヘルプを表示                                             [真偽]
-      --version  バージョンを表示                                         [真偽] 
+      --version  バージョンを表示                                         [真偽]
 ```
 ## クローリングの実行
 
@@ -76,7 +86,7 @@ APIキーは以下の３種類の方法で与える事ができます
 - 環境変数で渡す
   - ```API_KEY_plugin```という環境変数に格納する
   - コマンドに毎回含めなくて良くなる。
-  - **flickr**の場合は```.API_KEY_flickr```になります。
+  - **flickr**の場合は```API_KEY_flickr```になります。
     - ```plugin```はプラグイン名(flickr等)に置き換えてください。
   - 一時的な環境変数を定義する事も可能です。(bash等)
     - ```API_KEY_flickr="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" node crawler.js -p flickr -k "sea,ocean|mountain,mount" --vis-bulky```
@@ -90,7 +100,7 @@ APIキーは以下の３種類の方法で与える事ができます
 
 ### Bulky: 全ての点を地図上にポイントする
 
-![](assets/screenshot_venice_bulky.png)
+![](https://github.com/YokoyamaLab/Splatone/blob/main/assets/screenshot_venice_bulky.png?raw=true)
 
 * クエリは水域と通路・橋梁・ランドマークを色分けしたもの、上記スクリーンショットはベネチア付近のデータ
 ```shell
@@ -98,7 +108,7 @@ $ node crawler.js -p flickr -o '{"flickr":{"API_KEY":"aaaaaaaaaaaaaaaaaaaaaaaaaa
 ```
 
 ### Marker Cluster: 高密度の地点はマーカーをまとめて表示する
-![](assets/screenshot_venice_marker-cluster.png)
+![](https://github.com/YokoyamaLab/Splatone/blob/main/assets/screenshot_venice_marker-cluster.png?raw=true)
 
 * クエリは水域と通路・橋梁・ランドマークを色分けしたもの、上記スクリーンショットはベネチア付近のデータ
 ```shell
@@ -122,13 +132,6 @@ seaだけでは集められるポストが限定されるので、同様の意
 -k "sea,ocean|mountain,mount"
 ```
 
-### 実行例 (海岸線と山岳の分布)
-
-```shell
-$ node crawler.js -p flickr -o '{"flickr":{"API_KEY":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}' -k "sea,ocean|mountain,mount" --vis-bulky
-```
-![](assets/screenshot_sea-mountain_bulky.png)
-
 ### カテゴリ名の指定
 
 複数の類語キーワードを指定した場合、それらをまとめるカテゴリ名を付ける事ができます。たとえはsea,oceanに『海域』、mountain,mountに『山岳』とカテゴリ名をつけるには以下のように指定します。なお、指定は必須ではありません。指定しない場合はそれぞれ１番目のキーワード(seaとmountain)がカテゴリ名になります。
@@ -137,17 +140,25 @@ $ node crawler.js -p flickr -o '{"flickr":{"API_KEY":"aaaaaaaaaaaaaaaaaaaaaaaaaa
 -k "海域=sea,ocean|山岳=mountain,mount"
 ```
 
+### 実行例 (海岸線と山岳の分布)
+
+```shell
+$ node crawler.js -p flickr -o '{"flickr":{"API_KEY":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}' -k "sea,ocean|mountain,mount" --vis-bulky
+```
+![](https://github.com/YokoyamaLab/Splatone/blob/main/assets/screenshot_sea-mountain_bulky.png?raw=true)
+
+
 ## ダウンロード
 
 ### 画像のダウンロード
 
 * 結果の地図を画像(PNG形式)としてダウンロードするには、画面右下のアイコンをクリックしてください。
 
-![](assets/icon_image_download.png)
+![](https://github.com/YokoyamaLab/Splatone/blob/main/assets/icon_image_download.png?raw=true)
 
 ### データのダウンロード
 
 * クロール結果をデータとしてダウンロードしたい場合は凡例の下にあるエクスポートボタンをクリックしてください。
 
-![](assets/icon_data_export.png)
+![](https://github.com/YokoyamaLab/Splatone/blob/main/assets/icon_data_export.png?raw=true)
 

package/crawler.js

@@ -8,7 +8,7 @@ import os from 'node:os';
 import { EventEmitter } from 'node:events';
 import path, { resolve, dirname } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
-import { existsSync, constants } from 'node:fs';
+import { existsSync, writeFileSync, constants } from 'node:fs';
 import { access, readdir, readFile } from 'node:fs/promises';
 
 // -------------------------------
@@ -28,6 +28,7 @@ import { hideBin } from 'yargs/helpers';
 // -------------------------------
 import { loadPlugins } from './lib/pluginLoader.js';
 import paletteGenerator from './lib/paletteGenerator.js';
+import { dfsObject } from './lib/splatone.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -107,6 +108,7 @@ try {
   // コマンド例
   // node crawler.js -p flickr -o '{"flickr":{"API_KEY":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}' -k "商業=shop,souvenir,market,supermarket,pharmacy,store,department|食べ物=food,drink,restaurant,cafe,bar|美術 館=museum,art,exhibition,expo,sculpture,heritage|公園=park,garden,flower,green,pond,playground" --vis-bulky
   // node crawler.js -p flickr -k "水域=canal,channel,waterway,river,stream,watercourse,sea,ocean,gulf,bay,strait,lagoon,offshore|橋梁=bridge,overpass,flyover,aqueduct,trestle|通路=street,road,thoroughfare,roadway,avenue,boulevard,lane,alley,roadway,carriageway,highway,motorway|ランドマーク=church,sanctuary,chapel,cathedral,basilica,minster,abbey,temple,shrine" --vis-bulky
+  // node crawler.js -p flickr -k "水辺=sea,ocean,beach|山岳=mountain,mount,hill" --vis-bulky --chopped
   let yargv = await yargs(hideBin(process.argv))
     .strict()                        // 未定義オプションはエラー
     .usage('使い方: $0 [options]')
@@ -131,6 +133,22 @@ try {
       type: 'string',
       default: 'nature,tree,flower|building,house|water,sea,river,pond',
       description: '検索キーワード(|区切り)'
+    }).option('chopped', {
+      group: 'Basic Options',
+      alias: 'c',
+      type: 'boolean',
+      default: false,
+      description: '大きなデータを細分化して送信する'
+      //    }).option('debug-save', {
+      //      group: 'Debug',
+      //      type: 'boolean',
+      //      default: false,
+      //      description: 'サーバ側にクロールデータを保存'
+    }).option('debug-verbose', {
+      group: 'Debug',
+      type: 'boolean',
+      default: false,
+      description: 'デバッグ情報出力'
     })
     .version()
     .coerce({
@@ -154,7 +172,6 @@ try {
     return true;
   });
   const argv = await yargv.parseAsync();
-
   const visualizers = {};
   for (const vis of Object.keys(all_visualizers).filter(v => argv[`vis-${v}`])) {
     visualizers[vis] = new all_visualizers[vis]();
@@ -164,11 +181,15 @@ try {
   try {
     plugin_options.API_KEY = await loadAPIKey("flickr") ?? plugin_options.API_KEY;
   } catch (e) {
-    console.error("Error loading API key:", e.message);
+    if (!plugin_options.API_KEY) {
+      console.error("Error loading API key:", e.message);
+    }
     //Nothing to do
   }
   await plugins.call(argv.plugin, 'init', plugin_options);
-  console.table([["Visualizer", Object.keys(visualizers)], ["Plugin", argv.plugin]]);
+  if (argv.debugVerbose) {
+    console.table([["Visualizer", Object.keys(visualizers)], ["Plugin", argv.plugin]]);
+  }
 
   /* API Key読み込み */
   async function loadAPIKey(plugin = 'flickr') {
@@ -182,11 +203,11 @@ try {
       await access(file, constants.F_OK | constants.R_OK);
       // 読み込み & トリム
       const raw = await readFile(file, 'utf8');
-      console.log(`[API KEY (${plugin}})] Read from FILE`);
+      //console.log(`[API KEY (${plugin}})] Read from FILE`);
       key = raw.trim();
     } catch (err) {
       if (Object.prototype.hasOwnProperty.call(process.env, "API_KEY_" + plugin)) {
-        console.log(`[API KEY (${plugin}})] Read from ENV`);
+        //console.log(`[API KEY (${plugin}})] Read from ENV`);
         key = process.env["API_KEY_" + plugin] ?? null;
       } else {
         const code = /** @type {{ code?: string }} */(err).code || 'UNKNOWN';
@@ -325,7 +346,10 @@ try {
           console.warn("invalid sessionId:", req.sessionId);
           return;
         }
-        const { bbox, drawn, cellSize = '0.5', units = 'kilometers', tags = 'sea,beach|mountain,forest' } = req.query;
+        const { bbox, drawn, cellSize = 0, units = 'kilometers', tags = 'sea,beach|mountain,forest' } = req.query;
+        if (cellSize == 0) {
+          //セルサイズ自動決定()
+        }
         const fallbackBbox = [139.55, 35.53, 139.92, 35.80];
         let bboxArray = fallbackBbox;
 
@@ -527,7 +551,7 @@ try {
     return piscina.run(data, { filename });
   }
 
-  const nParallel = Math.max(1, Math.min(12, os.cpus().length))
+  const nParallel = Math.max(1, Math.min(6, os.cpus().length))
   const piscina = new Piscina({
     minThreads: 1,
     maxThreads: nParallel,
@@ -547,7 +571,9 @@ try {
     crawlers[p.sessionId][rtn.hexId][rtn.category].crawled ??= 0;
     crawlers[p.sessionId][rtn.hexId][rtn.category].total = rtn.final ? crawlers[p.sessionId][rtn.hexId][rtn.category].ids.size : rtn.total + crawlers[p.sessionId][rtn.hexId][rtn.category].crawled;
     crawlers[p.sessionId][rtn.hexId][rtn.category].crawled = crawlers[p.sessionId][rtn.hexId][rtn.category].ids.size;
-    console.log(`(CRAWL) ${rtn.hexId} ${rtn.category} ] dup=${duplicates.size}, out=${rtn.outside}, in=${rtn.photos.features.length}  || ${crawlers[p.sessionId][rtn.hexId][rtn.category].crawled} / ${crawlers[p.sessionId][rtn.hexId][rtn.category].total}`);
+    if (argv.debugVerbose) {
+      console.log(`(CRAWL) ${rtn.hexId} ${rtn.category} ] dup=${duplicates.size}, out=${rtn.outside}, in=${rtn.photos.features.length}  || ${crawlers[p.sessionId][rtn.hexId][rtn.category].crawled} / ${crawlers[p.sessionId][rtn.hexId][rtn.category].total}`);
+    }
     const photos = featureCollection(rtn.photos.features.filter((f) => !duplicates.has(f.properties.id)));
     crawlers[p.sessionId][rtn.hexId][rtn.category].items
       = concatFC(crawlers[p.sessionId][rtn.hexId][rtn.category].items, photos);
@@ -559,22 +585,97 @@ try {
       //console.log("next max_upload_date:", p.max_upload_date);
       api.emit('splatone:start', p);
     } else if (finish) {
-      console.table(stats);
+      if (argv.debugVerbose) {
+        console.table(stats);
+      }
       api.emit('splatone:finish', p);
     }
   });
 
   await subscribe('splatone:finish', async p => {
+    const resultId = uniqid();
     const result = crawlers[p.sessionId];
     const target = targets[p.sessionId];
+
+    let geoJson = Object.fromEntries(Object.entries(visualizers).map(([vis, v]) => [vis, v.getFutureCollection(result, target)]));
+
     console.log('[splatone:finish]');
-    const geoJson = Object.fromEntries(Object.entries(visualizers).map(([vis, v]) => [vis, v.getFutureCollection(result, target)]));
-    io.to(p.sessionId).emit('result', {
-      geoJson,
-      palette: target["splatonePalette"],
-      visualizers: Object.keys(visualizers),
-      plugin: argv.plugin
-    });
+    try {
+      if (argv.chopped) {
+        throw new RangeError("Invalid string length");
+      }
+      await io.to(p.sessionId).timeout(5000).emitWithAck('result', {
+        resultId,
+        geoJson,
+        palette: target["splatonePalette"],
+        visualizers: Object.keys(visualizers),
+        plugin: argv.plugin
+      });
+    } catch (e) {
+      if (e instanceof RangeError && /Invalid string length/.test(String(e.message))) {
+        const msg = (argv.chopped ? "ユーザの指定により" : "結果サイズが巨大なので") + "断片化モードでクライアントに送ります";
+        if (argv.debugVerbose) {
+          console.warn("[WARN] " + msg);
+        }
+        io.to(p.sessionId).timeout(5000).emit('toast', {
+          text: msg,
+          class: "warning"
+        });
+        //サイズ集計
+        const total_features = ((s = 0, st = [geoJson], v, seen = new WeakSet) => { for (; st.length;)if ((v = st.pop()) && typeof v === 'object' && !seen.has(v)) { seen.add(v); if (Array.isArray(v.features)) s += v.features.length; for (const k in v) { const x = v[k]; if (x && typeof x === 'object') st.push(x) } } return s })();
+        let current_features = 0
+        await dfsObject(geoJson, async ({ path, value, kind, type }) => {
+          if (path.length !== 0) {
+            if (kind === "primitive" || kind === "null") {
+              //console.log(path.join("."), "=>", `(${kind}:${type})`, value);
+              const ackrtn = await io.to(p.sessionId).timeout(5000).emitWithAck('result-chunk', {
+                resultId,
+                path,
+                kind,
+                type,
+                value
+              });
+              //console.log("\tACK", ackrtn);
+            } else if (kind === "object") {
+              //console.log(path.join("."), "=>", `(${kind}:${type})`);
+              if (path.at(-2) == "features" && Number.isInteger(path.at(-1))) {
+                current_features++;
+              }
+              const ackrtn = await io.to(p.sessionId).timeout(5000).emitWithAck('result-chunk', {
+                resultId,
+                path,
+                kind,
+                type,
+                progress: { current: current_features, total: total_features }
+              });
+              //console.log("\tACK", ackrtn);
+            } else if (kind === "array") {
+              //console.log(path.join("."), "=>", `(${kind}:${type})`);              
+              const ackrtn = await io.to(p.sessionId).timeout(5000).emitWithAck('result-chunk', {
+                resultId,
+                path,
+                kind,
+                type
+              });
+              //console.log("\tACK", ackrtn);
+            }
+          } else {
+            //console.log("SKIP---------------------");
+          }
+        });
+        //console.log("finish chunks");
+        io.to(p.sessionId).timeout(10000).emitWithAck('result', {
+          resultId,
+          geoJson: null, /*geoJsonは送らない*/
+          palette: target["splatonePalette"],
+          visualizers: Object.keys(visualizers),
+          plugin: argv.plugin
+        });
+        console.log("[Done]");
+      } else {
+        throw e; // 他の例外はそのまま
+      }
+    }
   });
 
   server.listen(port, async () => {

package/lib/paletteGenerator.js

@@ -61,7 +61,7 @@ const paletteGenerator = (function(undefined){
 			distanceType = 'Default';
 		ultra_precision = ultra_precision || false
 
-		console.log('Generate palettes for '+colorsCount+' colors using color distance "'+distanceType+'"')
+		//console.log('Generate palettes for '+colorsCount+' colors using color distance "'+distanceType+'"')
 
 		if(forceMode){
 			// Force Vector Mode

package/lib/splatone.js

@@ -0,0 +1,135 @@
+// 各種関数のまとめ場
+// @turf/turf v6/v7 どちらでもOK（rhumbDistanceはv7で統合済み）
+import { point, distance, rhumbDistance, bbox as turfBbox } from '@turf/turf';
+
+/**
+ * bbox の幅・高さを指定単位で返す
+ * @param {number[]} bbox [minX,minY,maxX,maxY] (経度・緯度, WGS84)
+ * @param {"meters"|"kilometers"|"miles"|"nauticalmiles"|"inches"|"yards"|"feet"} units
+ * @param {"geodesic"|"rhumb"} method  距離の定義：大円 or 等角航程
+ * @returns {{width:number, height:number, units:string}}
+ */
+export function bboxSize(bbox, units = "kilometers", method = "geodesic") {
+  const [minX, minY, maxX, maxY] = bbox;
+
+  // 中央経度・緯度（幅は中緯度、 高さは中経度で測るのが安定）
+  const midLat = (minY + maxY) / 2;
+  const midLon = (minX + maxX) / 2;
+
+  // 測りたい2点を作成
+  const west  = point([minX, midLat]);
+  const east  = point([maxX, midLat]);
+  const south = point([midLon, minY]);
+  const north = point([midLon, maxY]);
+
+  // 距離関数を選択
+  const distFn = method === "rhumb" ? rhumbDistance : distance;
+
+  return {
+    width:  distFn(west, east,  { units }),
+    height: distFn(south, north, { units }),
+    units
+  };
+}
+
+/**
+ * GeoJSON（Feature/FeatureCollection/Geometry）から計算したいときのヘルパー
+ */
+export function bboxSizeOf(geojson, units = "kilometers", method = "geodesic") {
+  return bboxSize(turfBbox(geojson), units, method);
+}
+
+
+// visit({ key, value, path, parent, depth, kind, type, isLeaf, isNull })
+//   kind: 'array' | 'object' | 'primitive' | 'null'
+//   type: primitive は typeof、配列は 'Array'、オブジェクトは constructor.name
+export async function dfsObject(root, visit, options = {}) {
+  const {
+    includeArrays = true,
+    maxDepth = Infinity,
+    stopOnTrue = false,
+  } = options;
+
+  const meta = describeType(root);
+
+  // root が非オブジェクトの場合も visit だけ呼んで終了
+  if (meta.isLeaf) {
+    await visit({
+      key: undefined,
+      value: root,
+      path: [],
+      parent: null,
+      depth: 0,
+      ...meta,
+    });
+    return false;
+  }
+
+  const seen = new WeakSet();
+  const stack = [
+    { key: undefined, value: root, path: /** @type {(string|number)[]} */([]), parent: null, depth: 0 }
+  ];
+
+  while (stack.length) {
+    const node = stack.pop();
+    const { key, value, path, parent } = node;
+
+    const m = describeType(value);
+    const depth = path.length; // path 配列の長さをそのまま深さに
+    const ret = await visit({ key, value, path, parent, depth, ...m });
+    if (stopOnTrue && ret === true) return true;
+
+    if (m.isLeaf) continue;                // primitive/null は展開しない
+    if (seen.has(value)) continue;         // 循環参照対策
+    seen.add(value);
+
+    if (depth >= maxDepth) continue;
+
+    const isArr = Array.isArray(value);
+    if (!includeArrays && isArr) continue; // 配列を辿らない設定
+
+    const entries = isArr
+      ? value.map((v, i) => [i, v])               // index は number
+      : Object.entries(value);                    // key は string
+
+    // 左→右の自然な DFS にしたいので逆順 push
+    for (let i = entries.length - 1; i >= 0; i--) {
+      const [k, v] = entries[i]; // k: string|number
+      const childPath = path.concat([k]);
+      stack.push({
+        key: k,
+        value: v,
+        path: childPath,
+        parent: value,
+        depth: childPath.length,
+      });
+    }
+  }
+  return false;
+}
+
+function describeType(value) {
+  if (value === null) {
+    return { kind: "null", type: "null", isLeaf: true, isNull: true };
+  }
+  const t = typeof value;
+  if (t !== "object") {
+    return {
+      kind: "primitive",
+      type: t, // 'number' | 'string' | 'boolean' | 'bigint' | 'symbol' | 'undefined'
+      isLeaf: true,
+      isNull: false,
+    };
+  }
+  if (Array.isArray(value)) {
+    return { kind: "array", type: "Array", isLeaf: false, isNull: false };
+  }
+  const ctor = value?.constructor?.name ?? "Object";
+  return { kind: "object", type: ctor, isLeaf: false, isNull: false };
+}
+
+
+export default {
+    dfsObject,bboxSize
+};
+//const {width,height} = bboxSize(b, "kilometers", "geodesic");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "splatone",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "description": "Multi-layer Composite Heatmap",
   "homepage": "https://github.com/YokoyamaLab/Splatone#readme",
   "bugs": {

package/plugins/flickr/worker.js

@@ -58,10 +58,11 @@ export default async function ({
         = res.photos.photo.length > 0
             ? (res.photos.photo[res.photos.photo.length - 1].dateupload) - (res.photos.photo[res.photos.photo.length - 1].dateupload == res.photos.photo[0].dateupload ? 1 : 0)
             : null;
-    if (Object.keys(authors).length == 1) {
-        const window = res.photos.photo[res.photos.photo.length - 1].dateupload - res.photos.photo[0].dateupload;
-        console.warn("[Warning]", `High posting activity detected for ${Object.keys(authors)} within ${window} s. the crawler will skip the next 24 hours.`);
-        next_max_upload_date -= 60 * 60 * 24;
+    const window = res.photos.photo.length == 0 ? 0 : res.photos.photo[0].dateupload - res.photos.photo[res.photos.photo.length - 1].dateupload;
+    if (Object.keys(authors).length == 1 && window < 60 * 60) {
+        const skip = window < 5 ? 0.1 : 12;
+        console.warn("[Warning]", `High posting activity detected for ${Object.keys(authors)} within ${window} s. the crawler will skip the next ${skip} hours.`);
+        next_max_upload_date -= 60 * 60 * skip;
     }
     return {
         photos,

package/public/style.css

@@ -315,4 +315,14 @@ text-shadow: 0 1px 1px #0006;
   .center-x {
     display: flex;
     justify-content: center;  /* 横中央 */
-  }
+  }
+
+.toast-info {
+    background: linear-gradient(to right, #43cea2, #185a9d)!important;
+}
+.toast-warning{
+    background: linear-gradient(to bottom, #c21500, #ffc500)!important;
+}
+.toast-error {
+     background: linear-gradient(to bottom, #333333, #dd1818)!important;
+}

package/public/visualizer.js

@@ -12,6 +12,21 @@ function addGeoJSONLayer(map, geojson, options = {}) {
     return geojsonLayer;
 }
 
+function setAt(obj, path, value) {
+    if (path.length === 0) return value;
+    let cur = obj;
+    for (let i = 0; i < path.length - 1; i++) {
+        const k = path[i];
+        const nextK = path[i + 1];
+        if (cur[k] == null) {
+            // 次のキーが数値なら []、文字なら {}
+            cur[k] = Number.isInteger(nextK) ? [] : {};
+        }
+        cur = cur[k];
+    }
+    cur[path[path.length - 1]] = value;
+    return obj;
+}
 /*
 使用例:
 const layer = addGeoJSONLayer(map, myGeoJSON, {

package/views/index.ejs

@@ -11,7 +11,9 @@
   <link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" crossorigin="anonymous" />
   <!-- Leaflet.draw CSS -->
   <link rel="stylesheet" href="https://unpkg.com/leaflet-draw@1.0.4/dist/leaflet.draw.css" crossorigin="anonymous" />
-  <link rel="stylesheet" href="/style.css" />
+  <link rel="stylesheet" href="style.css" />
+  <!-- Toastify -->
+  <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/toastify-js/src/toastify.min.css">
 </head>
 
 <body>
@@ -99,7 +101,8 @@
   <!-- Map to Image -->
   <script src="https://cdn.jsdelivr.net/npm/leaflet-easyprint@2.1.9/dist/bundle.min.js"></script>
   <link href="https://cdn.jsdelivr.net/npm/leaflet-easyprint@2.1.9/libs/leaflet.min.css" rel="stylesheet">
-
+  <!-- Toastify -->
+  <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/toastify-js"></script>
   <!-- Socket.IO JS -->
   <script src="https://cdn.socket.io/4.7.5/socket.io.min.js"></script>
   <!-- Visualize.js -->
@@ -145,15 +148,20 @@
       //console.log('Disconnected from server');
     });
 
-    let sessionId=null;
+    let sessionId = null;
     let visualizers = {};
-    let layerControl=null;
-    socket.on("welcome",async (res) => {
+    let results = {};
+    let layerControl = null;
+    socket.on("welcome", async (res) => {
       //console.log(`welcome ${res.sessionId} at ${new Date(res.time).toLocaleTimeString()}`);
-      console.log("VIS",res.visualizers);
+      //console.log("VIS", res.visualizers);
       sessionId = res.sessionId;
-      visualizers = Object.fromEntries(await Promise.all(res.visualizers.map(async vis => { const m = await import(new URL(`./visualizer/${vis}/web.js`, import.meta.url).href); return [vis, m.entry ?? m.default ?? m[vis]]; })));
-      console.log(visualizers);
+      visualizers = Object.fromEntries(await Promise.all(res.visualizers.map(async vis => {
+        const m = await import(new URL(`./visualizer/${vis}/web.js`,
+          import.meta.url).href);
+        return [vis, m.entry ?? m.default ?? m[vis]];
+      })));
+      //console.log(visualizers);
     });
 
     function generateLegend(legends) {
@@ -222,46 +230,102 @@
           crawled: acc.crawled + (g.crawled ?? 0),
           total: acc.total + (g.total ?? 0)
         }), {
-        crawled: 0,
-        total: 0
-      }
+          crawled: 0,
+          total: 0
+        }
       );
       if (total == 0) {
         crawled = 1, total = 1;
       }
-      const percent = Math.round((crawled / total) * 100);
-      console.log(`[Done] ${crawled} / ${total} --> ${percent}`);
+      const percent = Math.round((crawled / total) * 100000) / 1000;
+      //console.log(`[Done] ${crawled} / ${total} --> ${percent}`);
       setProgress(document.getElementById("progressCrawl"), percent);
       //ここでHexのプログレスグラデーションレイヤのOpacity調整
       highlightHexById(res.hexId, 1 - res.progress[res.hexId].percent, "#263238");
     });
 
     function isPlainObject(a) {
-      if (a === null || typeof a !== 'object') return false;  // 原始値/関数など除外
+      if (a === null || typeof a !== 'object') return false; // 原始値/関数など除外
       const proto = Object.getPrototypeOf(a);
-      return proto === Object.prototype || proto === null;     // {} / Object.create(null)
+      return proto === Object.prototype || proto === null; // {} / Object.create(null)
     }
+    socket.on("toast", (arg) => {
+      const option = {
+        text:"",
+        duration:arg.close?-1:5000,
+        close:false,
+        gravity:'bottom',
+        position:'center',
+        className:"toast-" + arg.class,
+        ...arg};
+      Toastify(option).showToast();
+      //https://apvarun.github.io/toastify-js/#
+    });
+    socket.on("result-chunk", (arg, callback) => {
+      if(arg.progress){
+        setProgress(document.getElementById("progressCrawl"),Math.round((arg.progress.current/arg.progress.total) * 100000) / 1000);
+      }
+      results[arg.resultId] ??= {};
+      if (arg.kind === "primitive" || arg.kind === "null") {
+        //'number' | 'string' | 'boolean' | 'bigint' | 'symbol' | 'undefined'
+        //console.log(arg.path,arg.value);
+        switch (arg.type) {
+          case 'number':
+            setAt(results[arg.resultId], arg.path, Number(arg.value));
+            break;
+          case 'boolean':
+            setAt(results[arg.resultId], arg.path, arg.value);
+          case 'string':
+            setAt(results[arg.resultId], arg.path, String(arg.value));
+            break;
+          case 'bigint':
+            setAt(results[arg.resultId], arg.path, BigInt(arg.value));
+            break;
+          case 'symbol':
+            setAt(results[arg.resultId], arg.path, Symbol(arg.value));
+            break;
+          default:
+            setAt(results[arg.resultId], arg.path, null);
+        }
+      } else if (arg.kind === "object") {
+        setAt(results[arg.resultId], arg.path, {});
+      } else if (arg.kind === "array") {
+        setAt(results[arg.resultId], arg.path, []);
+      }
+      //console.log("ARK");
+      callback({ pong: Date.now(), ok: true }); // only one argument is expected
+    });
 
     let latestResult = {};
     const overlays = {};
-    socket.on('result',async (res) => {
+    socket.on('result', async (res,callback) => {
+      //console.log("result");
       latestResult["_"]["visualizers"] = res.visualizers;
       latestResult["_"]["plugin"] = res.plugin;
       clearHexHighlight();
-      latestResult = {...latestResult, ...res.geoJson};
-      console.log("result");
-      console.log("geoJSON",res.geoJson);
-      for(const vis in visualizers){
-        const layers = await visualizers[vis](map, res.geoJson[vis], {palette:res.palette});
-        console.log("【レイヤ】",layers);
-        if((layers == null) ){
+      if(res.geoJson == null){
+        res.geoJson = results[res.resultId];
+      }else{
+        results[res.resultId] = res.geoJson;
+      }
+      latestResult = {
+        ...latestResult,
+        ...res.geoJson
+      };
+      for (const vis in visualizers) {
+        const layers = await visualizers[vis](map, res.geoJson[vis], {
+          palette: res.palette
+        });
+        //console.log("【レイヤ】\n", JSON.stringify(layers,null,4));
+        if ((layers == null)) {
           //SKIP
-        }else if(layers.hasOwnProperty("type") && layers["type"]=="FeatureCollection"){
+        } else if (layers.hasOwnProperty("type") && layers["type"] == "FeatureCollection") {
           //レイヤ一つ
-          overlays[`[${vis}]`] =  layers;
-        }else{
-          for(const name in layers){
-            overlays[`[${vis}] ${name}`] =  layers[name];
+          overlays[`[${vis}]`] = layers;
+        } else {
+          for (const name in layers) {
+            console.log("Lay",vis,name);
+            overlays[`[${vis}] ${name}`] = layers[name];
           }
         }
       }
@@ -280,7 +344,7 @@
         });
       }
       ul.innerHTML = generateLegend(palette);
-      console.log(overlays);
+      //console.log(overlays);
       layerControl = L.control.layers([baseLayer], {
         ...overlays,
         "Hex Grid": hexLayer,
@@ -295,11 +359,12 @@
         downloadJSON(`data-${stamp}.json`, latestResult);
         let sessionId = null;
       });
+      callback({ pong: Date.now(), ok: true });
     });
 
     // 地図
     const map = L.map('map', {
-      preferCanvas:true,
+      preferCanvas: true,
       zoomControl: true
     }).setView([lat, lon], 12);
     const baseLayer = L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', {
@@ -348,11 +413,11 @@
     L.easyPrint({
       title: 'マップの画像化とダウンロード',
       position: 'bottomright',
-      sizeModes: [ 'A4Landscape'],
+      sizeModes: ['A4Landscape'],
       exportOnly: true,
       filename: 'splatone_export'
     }).addTo(map);
-    
+
     // レイヤ参照
     let hexLayer = null;
     let triLayer = null;
@@ -603,7 +668,7 @@
       const cellSize = document.getElementById('cellSize').value.trim();
       const units = document.getElementById('units').value.trim();
       const keywords = document.getElementById('keywords').value.trim();
-      latestResult["_"]??={};
+      latestResult["_"] ??= {};
       latestResult["_"]["keywords"] = keywords;
       socket.once("hexgrid", (res) => {
         if (res.sessionId) sessionId = res.sessionId;
@@ -611,7 +676,7 @@
         const hex = res.hex;
         const triangles = res.triangles;
         latestResult["_"]["hex"] = hex;
-        latestResult["_"]["triangles"]=[triangles];
+        latestResult["_"]["triangles"] = [triangles];
         renderLayers(hex, triangles);
       });
       socket.emit("target", {

package/visualizer/bulky/node.js

@@ -1,6 +1,5 @@
 import { VisualizerBase } from '../../lib/VisualizerBase.js';
 import { featureCollection } from "@turf/turf";
-import { writeFileSync } from 'node:fs';
 
 export default class BulkyVisualizer extends VisualizerBase {
     static id = 'bulky';            // 一意ID（フォルダ名と一致させると運用しやすい）

package/visualizer/bulky/web.js

@@ -1,9 +1,11 @@
 let booted = false;
 export default async function main(map, geojson, options = {}) {
+    console.log("main");
     if (booted) return;
     booted = true;
     const layers = {};
     for (const cat in geojson) {
+        console.log(cat);
         const layer = addGeoJSONLayer(map, geojson[cat], {
             pointToLayer: (feature, latlng) => {
                 return L.circleMarker(latlng, {