splatone 0.0.4 → 0.0.6

package/README.md

@@ -13,8 +13,13 @@ SNSのジオタグ付きポストをキーワードに基づいて収集する
 
 ## 既知のバグ
 
-- JSON.stringify(json)で変換できる大きさに制限があり、数十万件等の大きな結果を生み出すクエリは、クロール後、結果のブラウザへの転送で失敗します。
-  
+- <s>JSON.stringify(json)で変換できる大きさに制限があり、数十万件等の大きな結果を生み出すクエリは、クロール後、結果のブラウザへの転送で失敗します。</s>
+- サイズの問題は、一度ファイルに書き出す事で解決しました。(v0.0.6)
+  - **--no-filed**オプションを付ける事で、従来の方法で実行できます。(ファイルに書き出すより多少速い)
+- 以下の画像のように大きな範囲で多くのジオタグが収集できるようになりました。
+
+![](/assets/screenshot_massive_points_bulky.png)
+
 # 使い方
 
 - [Node.js](https://nodejs.org/ja/download)をインストール後、npxで実行します。
@@ -25,13 +30,18 @@ SNSのジオタグ付きポストをキーワードに基づいて収集する
 ```shell
 $ npx -y -- splatone@latest crawler --help
 使い方: crawler.js [options]
-
 Basic Options
-  -p, --plugin    実行するプラグイン
-                                   [文字列] [必須] [選択してください: "flickr"]
+  -p, --plugin    実行するプラグイン[文字列] [必須] [選択してください: "flickr"]
   -o, --options   プラグインオプション               [文字列] [デフォルト: "{}"]
   -k, --keywords  検索キーワード(|区切り)                  [文字列] [デフォルト:
                        "nature,tree,flower|building,house|water,sea,river,pond"]
+  -f, --filed     大きなデータをファイルとして送受信する
+                                                       [真偽] [デフォルト: true]
+  -c, --chopped   大きなデータを細分化して送受信する
+                                             [非推奨] [真偽] [デフォルト: false]
+
+Debug
+      --debug-verbose  デバッグ情報出力               [真偽] [デフォルト: false]
 
 Visualization (最低一つの指定が必須です)
       --vis-bulky           全データをCircleMarkerとして地図上に表示
@@ -41,7 +51,7 @@ Visualization (最低一つの指定が必須です)
 
 オプション:
       --help     ヘルプを表示                                             [真偽]
-      --version  バージョンを表示                                         [真偽] 
+      --version  バージョンを表示                                         [真偽]
 ```
 ## クローリングの実行
 
@@ -76,7 +86,7 @@ APIキーは以下の３種類の方法で与える事ができます
 - 環境変数で渡す
   - ```API_KEY_plugin```という環境変数に格納する
   - コマンドに毎回含めなくて良くなる。
-  - **flickr**の場合は```.API_KEY_flickr```になります。
+  - **flickr**の場合は```API_KEY_flickr```になります。
     - ```plugin```はプラグイン名(flickr等)に置き換えてください。
   - 一時的な環境変数を定義する事も可能です。(bash等)
     - ```API_KEY_flickr="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" node crawler.js -p flickr -k "sea,ocean|mountain,mount" --vis-bulky```

package/crawler.js

@@ -8,7 +8,7 @@ import os from 'node:os';
 import { EventEmitter } from 'node:events';
 import path, { resolve, dirname } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
-import { existsSync, constants } from 'node:fs';
+import { existsSync, writeFileSync, constants } from 'node:fs';
 import { access, readdir, readFile } from 'node:fs/promises';
 
 // -------------------------------
@@ -28,6 +28,7 @@ import { hideBin } from 'yargs/helpers';
 // -------------------------------
 import { loadPlugins } from './lib/pluginLoader.js';
 import paletteGenerator from './lib/paletteGenerator.js';
+import { dfsObject, saveGeoJsonObjectAsStream } from './lib/splatone.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -107,6 +108,7 @@ try {
   // コマンド例
   // node crawler.js -p flickr -o '{"flickr":{"API_KEY":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}' -k "商業=shop,souvenir,market,supermarket,pharmacy,store,department|食べ物=food,drink,restaurant,cafe,bar|美術 館=museum,art,exhibition,expo,sculpture,heritage|公園=park,garden,flower,green,pond,playground" --vis-bulky
   // node crawler.js -p flickr -k "水域=canal,channel,waterway,river,stream,watercourse,sea,ocean,gulf,bay,strait,lagoon,offshore|橋梁=bridge,overpass,flyover,aqueduct,trestle|通路=street,road,thoroughfare,roadway,avenue,boulevard,lane,alley,roadway,carriageway,highway,motorway|ランドマーク=church,sanctuary,chapel,cathedral,basilica,minster,abbey,temple,shrine" --vis-bulky
+  // node crawler.js -p flickr -k "水辺=sea,ocean,beach|山岳=mountain,mount,hill" --vis-bulky --chopped
   let yargv = await yargs(hideBin(process.argv))
     .strict()                        // 未定義オプションはエラー
     .usage('使い方: $0 [options]')
@@ -131,6 +133,29 @@ try {
       type: 'string',
       default: 'nature,tree,flower|building,house|water,sea,river,pond',
       description: '検索キーワード(|区切り)'
+    }).option('filed', {
+      group: 'Basic Options',
+      alias: 'f',
+      type: 'boolean',
+      default: true,
+      description: '大きなデータをファイルとして送受信する'
+    }).option('chopped', {
+      group: 'Basic Options',
+      alias: 'c',
+      type: 'boolean',
+      default: false,
+      deprecate: true,
+      description: '大きなデータを細分化して送受信する'
+      //    }).option('debug-save', {
+      //      group: 'Debug',
+      //      type: 'boolean',
+      //      default: false,
+      //      description: 'サーバ側にクロールデータを保存'
+    }).option('debug-verbose', {
+      group: 'Debug',
+      type: 'boolean',
+      default: false,
+      description: 'デバッグ情報出力'
     })
     .version()
     .coerce({
@@ -151,10 +176,13 @@ try {
     if (Object.keys(all_visualizers).filter(v => argv["vis-" + v]).length == 0) {
       throw new Error('可視化ツールの指定がありません。最低一つは指定してください。');
     }
+    if (argv.filed && argv.chopped) {
+      console.warn("--filedと--choppedが両方指定されています。--filedが優先されます。");
+      argv.chopped = false;
+    }
     return true;
   });
   const argv = await yargv.parseAsync();
-
   const visualizers = {};
   for (const vis of Object.keys(all_visualizers).filter(v => argv[`vis-${v}`])) {
     visualizers[vis] = new all_visualizers[vis]();
@@ -164,11 +192,15 @@ try {
   try {
     plugin_options.API_KEY = await loadAPIKey("flickr") ?? plugin_options.API_KEY;
   } catch (e) {
-    console.error("Error loading API key:", e.message);
+    if (!plugin_options.API_KEY) {
+      console.error("Error loading API key:", e.message);
+    }
     //Nothing to do
   }
   await plugins.call(argv.plugin, 'init', plugin_options);
-  console.table([["Visualizer", Object.keys(visualizers)], ["Plugin", argv.plugin]]);
+  if (argv.debugVerbose) {
+    console.table([["Visualizer", Object.keys(visualizers)], ["Plugin", argv.plugin]]);
+  }
 
   /* API Key読み込み */
   async function loadAPIKey(plugin = 'flickr') {
@@ -182,11 +214,11 @@ try {
       await access(file, constants.F_OK | constants.R_OK);
       // 読み込み & トリム
       const raw = await readFile(file, 'utf8');
-      console.log(`[API KEY (${plugin}})] Read from FILE`);
+      //console.log(`[API KEY (${plugin}})] Read from FILE`);
       key = raw.trim();
     } catch (err) {
       if (Object.prototype.hasOwnProperty.call(process.env, "API_KEY_" + plugin)) {
-        console.log(`[API KEY (${plugin}})] Read from ENV`);
+        //console.log(`[API KEY (${plugin}})] Read from ENV`);
         key = process.env["API_KEY_" + plugin] ?? null;
       } else {
         const code = /** @type {{ code?: string }} */(err).code || 'UNKNOWN';
@@ -325,7 +357,10 @@ try {
           console.warn("invalid sessionId:", req.sessionId);
           return;
         }
-        const { bbox, drawn, cellSize = '0.5', units = 'kilometers', tags = 'sea,beach|mountain,forest' } = req.query;
+        const { bbox, drawn, cellSize = 0, units = 'kilometers', tags = 'sea,beach|mountain,forest' } = req.query;
+        if (cellSize == 0) {
+          //セルサイズ自動決定()
+        }
         const fallbackBbox = [139.55, 35.53, 139.92, 35.80];
         let bboxArray = fallbackBbox;
 
@@ -527,7 +562,7 @@ try {
     return piscina.run(data, { filename });
   }
 
-  const nParallel = Math.max(1, Math.min(12, os.cpus().length))
+  const nParallel = Math.max(1, Math.min(6, os.cpus().length))
   const piscina = new Piscina({
     minThreads: 1,
     maxThreads: nParallel,
@@ -547,7 +582,9 @@ try {
     crawlers[p.sessionId][rtn.hexId][rtn.category].crawled ??= 0;
     crawlers[p.sessionId][rtn.hexId][rtn.category].total = rtn.final ? crawlers[p.sessionId][rtn.hexId][rtn.category].ids.size : rtn.total + crawlers[p.sessionId][rtn.hexId][rtn.category].crawled;
     crawlers[p.sessionId][rtn.hexId][rtn.category].crawled = crawlers[p.sessionId][rtn.hexId][rtn.category].ids.size;
-    console.log(`(CRAWL) ${rtn.hexId} ${rtn.category} ] dup=${duplicates.size}, out=${rtn.outside}, in=${rtn.photos.features.length}  || ${crawlers[p.sessionId][rtn.hexId][rtn.category].crawled} / ${crawlers[p.sessionId][rtn.hexId][rtn.category].total}`);
+    if (argv.debugVerbose) {
+      console.log(`(CRAWL) ${rtn.hexId} ${rtn.category} ] dup=${duplicates.size}, out=${rtn.outside}, in=${rtn.photos.features.length}  || ${crawlers[p.sessionId][rtn.hexId][rtn.category].crawled} / ${crawlers[p.sessionId][rtn.hexId][rtn.category].total}`);
+    }
     const photos = featureCollection(rtn.photos.features.filter((f) => !duplicates.has(f.properties.id)));
     crawlers[p.sessionId][rtn.hexId][rtn.category].items
       = concatFC(crawlers[p.sessionId][rtn.hexId][rtn.category].items, photos);
@@ -559,22 +596,111 @@ try {
       //console.log("next max_upload_date:", p.max_upload_date);
       api.emit('splatone:start', p);
     } else if (finish) {
-      console.table(stats);
+      if (argv.debugVerbose) {
+        console.table(stats);
+      }
       api.emit('splatone:finish', p);
     }
   });
 
   await subscribe('splatone:finish', async p => {
+    const resultId = uniqid();
     const result = crawlers[p.sessionId];
     const target = targets[p.sessionId];
+
+    let geoJson = Object.fromEntries(Object.entries(visualizers).map(([vis, v]) => [vis, v.getFutureCollection(result, target)]));
+
     console.log('[splatone:finish]');
-    const geoJson = Object.fromEntries(Object.entries(visualizers).map(([vis, v]) => [vis, v.getFutureCollection(result, target)]));
-    io.to(p.sessionId).emit('result', {
-      geoJson,
-      palette: target["splatonePalette"],
-      visualizers: Object.keys(visualizers),
-      plugin: argv.plugin
-    });
+    try {
+      if (argv.chopped || argv.filed) {
+        throw new RangeError("Invalid string length");
+      }
+      await io.to(p.sessionId).timeout(120000).emitWithAck('result', {
+        resultId,
+        geoJson,
+        palette: target["splatonePalette"],
+        visualizers: Object.keys(visualizers),
+        plugin: argv.plugin
+      });
+    } catch (e) {
+      if (e instanceof RangeError && /Invalid string length/.test(String(e.message))) {
+        const msg = ((argv.chopped || argv.filed) ? "ユーザの指定により" : "結果サイズが巨大なので") + (argv.chopped ? "断片化送信" : "保存ファイル送信") + "モードでクライアントに送ります";
+        if (argv.debugVerbose) {
+          console.warn("[WARN] " + msg);
+        }
+        io.to(p.sessionId).timeout(5000).emit('toast', {
+          text: msg,
+          class: "warning"
+        });
+        if (argv.chopped) {
+          //サイズ集計
+          const total_features = ((s = 0, st = [geoJson], v, seen = new WeakSet) => { for (; st.length;)if ((v = st.pop()) && typeof v === 'object' && !seen.has(v)) { seen.add(v); if (Array.isArray(v.features)) s += v.features.length; for (const k in v) { const x = v[k]; if (x && typeof x === 'object') st.push(x) } } return s })();
+          let current_features = 0
+          await dfsObject(geoJson, async ({ path, value, kind, type }) => {
+            if (path.length !== 0) {
+              if (kind === "primitive" || kind === "null") {
+                //console.log(path.join("."), "=>", `(${kind}:${type})`, value);
+                const ackrtn = await io.to(p.sessionId).timeout(120000).emitWithAck('result-chunk', {
+                  resultId,
+                  path,
+                  kind,
+                  type,
+                  value
+                });
+                //console.log("\tACK", ackrtn);
+              } else if (kind === "object") {
+                //console.log(path.join("."), "=>", `(${kind}:${type})`);
+                if (path.at(-2) == "features" && Number.isInteger(path.at(-1))) {
+                  current_features++;
+                }
+                const ackrtn = await io.to(p.sessionId).timeout(120000).emitWithAck('result-chunk', {
+                  resultId,
+                  path,
+                  kind,
+                  type,
+                  progress: { current: current_features, total: total_features }
+                });
+                //console.log("\tACK", ackrtn);
+              } else if (kind === "array") {
+                //console.log(path.join("."), "=>", `(${kind}:${type})`);              
+                const ackrtn = await io.to(p.sessionId).timeout(120000).emitWithAck('result-chunk', {
+                  resultId,
+                  path,
+                  kind,
+                  type
+                });
+                //console.log("\tACK", ackrtn);
+              }
+            } else {
+              //console.log("SKIP---------------------");
+            }
+          });
+          //console.log("finish chunks");
+        } else {
+          //保存ファイル送信(--filed)
+          try {
+            const outPath = await saveGeoJsonObjectAsStream(geoJson, 'result.' + resultId + '.json');
+            console.log('saved:', outPath);
+            const ackrtn = await io.to(p.sessionId).timeout(120000).emitWithAck('result-file', {
+              resultId,
+            });
+          } catch (err) {
+            console.error('failed:', err);
+            process.exitCode = 1;
+          }
+        }
+        await io.to(p.sessionId).timeout(120000).emitWithAck('result', {
+          resultId,
+          geoJson: null, /*geoJsonは送らない*/
+          palette: target["splatonePalette"],
+          visualizers: Object.keys(visualizers),
+          plugin: argv.plugin
+        });
+        console.log("[Done]");
+      } else {
+        throw e; // 他の例外はそのまま
+      }
+    }
   });
 
   server.listen(port, async () => {

package/lib/paletteGenerator.js

@@ -61,7 +61,7 @@ const paletteGenerator = (function(undefined){
 			distanceType = 'Default';
 		ultra_precision = ultra_precision || false
 
-		console.log('Generate palettes for '+colorsCount+' colors using color distance "'+distanceType+'"')
+		//console.log('Generate palettes for '+colorsCount+' colors using color distance "'+distanceType+'"')
 
 		if(forceMode){
 			// Force Vector Mode

package/lib/splatone.js

@@ -0,0 +1,151 @@
+// 各種関数のまとめ場
+// @turf/turf v6/v7 どちらでもOK（rhumbDistanceはv7で統合済み）
+import { point, distance, rhumbDistance, bbox as turfBbox } from '@turf/turf';
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import path from 'node:path';
+import { pipeline as pipelineCb } from 'node:stream';
+import { promisify } from 'node:util';
+import { JsonStreamStringify } from 'json-stream-stringify';
+
+/**
+ * bbox の幅・高さを指定単位で返す
+ * @param {number[]} bbox [minX,minY,maxX,maxY] (経度・緯度, WGS84)
+ * @param {"meters"|"kilometers"|"miles"|"nauticalmiles"|"inches"|"yards"|"feet"} units
+ * @param {"geodesic"|"rhumb"} method  距離の定義：大円 or 等角航程
+ * @returns {{width:number, height:number, units:string}}
+ */
+export function bboxSize(bbox, units = "kilometers", method = "geodesic") {
+  const [minX, minY, maxX, maxY] = bbox;
+
+  // 中央経度・緯度（幅は中緯度、 高さは中経度で測るのが安定）
+  const midLat = (minY + maxY) / 2;
+  const midLon = (minX + maxX) / 2;
+
+  // 測りたい2点を作成
+  const west = point([minX, midLat]);
+  const east = point([maxX, midLat]);
+  const south = point([midLon, minY]);
+  const north = point([midLon, maxY]);
+
+  // 距離関数を選択
+  const distFn = method === "rhumb" ? rhumbDistance : distance;
+
+  return {
+    width: distFn(west, east, { units }),
+    height: distFn(south, north, { units }),
+    units
+  };
+}
+
+/**
+ * GeoJSON（Feature/FeatureCollection/Geometry）から計算したいときのヘルパー
+ */
+export function bboxSizeOf(geojson, units = "kilometers", method = "geodesic") {
+  return bboxSize(turfBbox(geojson), units, method);
+}
+
+
+// visit({ key, value, path, parent, depth, kind, type, isLeaf, isNull })
+//   kind: 'array' | 'object' | 'primitive' | 'null'
+//   type: primitive は typeof、配列は 'Array'、オブジェクトは constructor.name
+export async function dfsObject(root, visit, options = {}) {
+  const {
+    includeArrays = true,
+    maxDepth = Infinity,
+    stopOnTrue = false,
+  } = options;
+
+  const meta = describeType(root);
+
+  // root が非オブジェクトの場合も visit だけ呼んで終了
+  if (meta.isLeaf) {
+    await visit({
+      key: undefined,
+      value: root,
+      path: [],
+      parent: null,
+      depth: 0,
+      ...meta,
+    });
+    return false;
+  }
+
+  const seen = new WeakSet();
+  const stack = [
+    { key: undefined, value: root, path: /** @type {(string|number)[]} */([]), parent: null, depth: 0 }
+  ];
+
+  while (stack.length) {
+    const node = stack.pop();
+    const { key, value, path, parent } = node;
+
+    const m = describeType(value);
+    const depth = path.length; // path 配列の長さをそのまま深さに
+    const ret = await visit({ key, value, path, parent, depth, ...m });
+    if (stopOnTrue && ret === true) return true;
+
+    if (m.isLeaf) continue;                // primitive/null は展開しない
+    if (seen.has(value)) continue;         // 循環参照対策
+    seen.add(value);
+
+    if (depth >= maxDepth) continue;
+
+    const isArr = Array.isArray(value);
+    if (!includeArrays && isArr) continue; // 配列を辿らない設定
+
+    const entries = isArr
+      ? value.map((v, i) => [i, v])               // index は number
+      : Object.entries(value);                    // key は string
+
+    // 左→右の自然な DFS にしたいので逆順 push
+    for (let i = entries.length - 1; i >= 0; i--) {
+      const [k, v] = entries[i]; // k: string|number
+      const childPath = path.concat([k]);
+      stack.push({
+        key: k,
+        value: v,
+        path: childPath,
+        parent: value,
+        depth: childPath.length,
+      });
+    }
+  }
+  return false;
+}
+
+function describeType(value) {
+  if (value === null) {
+    return { kind: "null", type: "null", isLeaf: true, isNull: true };
+  }
+  const t = typeof value;
+  if (t !== "object") {
+    return {
+      kind: "primitive",
+      type: t, // 'number' | 'string' | 'boolean' | 'bigint' | 'symbol' | 'undefined'
+      isLeaf: true,
+      isNull: false,
+    };
+  }
+  if (Array.isArray(value)) {
+    return { kind: "array", type: "Array", isLeaf: false, isNull: false };
+  }
+  const ctor = value?.constructor?.name ?? "Object";
+  return { kind: "object", type: ctor, isLeaf: false, isNull: false };
+}
+const pipeline = promisify(pipelineCb);
+export async function saveGeoJsonObjectAsStream(geoJsonObject, outfile) {
+  const dir = path.join('public', 'out');
+  await mkdir(dir, { recursive: true });
+  const destPath = path.join(dir, path.basename(outfile));
+
+  // ここでも JSON.stringify は呼ばず、ストリームで直に文字列化
+  const src = new JsonStreamStringify(geoJsonObject);
+  const dest = createWriteStream(destPath, { flags: 'w' });
+  await pipeline(src, dest);
+  return destPath;
+}
+export default {
+  dfsObject, bboxSize, saveGeoJsonObjectAsStream
+};
+//const {width,height} = bboxSize(b, "kilometers", "geodesic");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "splatone",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "Multi-layer Composite Heatmap",
   "homepage": "https://github.com/YokoyamaLab/Splatone#readme",
   "bugs": {
@@ -27,6 +27,7 @@
     "express": "^5.1.0",
     "flickr-sdk": "^7.1.0",
     "iwanthue": "^2.0.0",
+    "json-stream-stringify": "^3.1.6",
     "open": "^10.2.0",
     "piscina": "^5.1.3",
     "socket.io": "^4.8.1",

package/plugins/flickr/worker.js

@@ -58,10 +58,11 @@ export default async function ({
         = res.photos.photo.length > 0
             ? (res.photos.photo[res.photos.photo.length - 1].dateupload) - (res.photos.photo[res.photos.photo.length - 1].dateupload == res.photos.photo[0].dateupload ? 1 : 0)
             : null;
-    if (Object.keys(authors).length == 1) {
-        const window = res.photos.photo[res.photos.photo.length - 1].dateupload - res.photos.photo[0].dateupload;
-        console.warn("[Warning]", `High posting activity detected for ${Object.keys(authors)} within ${window} s. the crawler will skip the next 24 hours.`);
-        next_max_upload_date -= 60 * 60 * 24;
+    const window = res.photos.photo.length == 0 ? 0 : res.photos.photo[0].dateupload - res.photos.photo[res.photos.photo.length - 1].dateupload;
+    if (Object.keys(authors).length == 1 && window < 60 * 60) {
+        const skip = window < 5 ? 0.1 : 12;
+        console.warn("[Warning]", `High posting activity detected for ${Object.keys(authors)} within ${window} s. the crawler will skip the next ${skip} hours.`);
+        next_max_upload_date -= 60 * 60 * skip;
     }
     return {
         photos,

package/public/style.css

@@ -315,4 +315,14 @@ text-shadow: 0 1px 1px #0006;
   .center-x {
     display: flex;
     justify-content: center;  /* 横中央 */
-  }
+  }
+
+.toast-info {
+    background: linear-gradient(to right, #43cea2, #185a9d)!important;
+}
+.toast-warning{
+    background: linear-gradient(to bottom, #c21500, #ffc500)!important;
+}
+.toast-error {
+     background: linear-gradient(to bottom, #333333, #dd1818)!important;
+}

package/public/visualizer.js

@@ -12,6 +12,21 @@ function addGeoJSONLayer(map, geojson, options = {}) {
     return geojsonLayer;
 }
 
+function setAt(obj, path, value) {
+    if (path.length === 0) return value;
+    let cur = obj;
+    for (let i = 0; i < path.length - 1; i++) {
+        const k = path[i];
+        const nextK = path[i + 1];
+        if (cur[k] == null) {
+            // 次のキーが数値なら []、文字なら {}
+            cur[k] = Number.isInteger(nextK) ? [] : {};
+        }
+        cur = cur[k];
+    }
+    cur[path[path.length - 1]] = value;
+    return obj;
+}
 /*
 使用例:
 const layer = addGeoJSONLayer(map, myGeoJSON, {
@@ -46,4 +61,77 @@ function loadAsset(url) {
     }
 
     return Promise.reject(new Error(`Unsupported extension: ${url}`));
-}
+}
+
+async function fetchJsonObject(url, { signal } = {}) {
+    const res = await fetch(url, {
+        method: 'GET',
+        headers: { 'Accept': 'application/json' },
+        // same-originのときは不要。CORS越えならサーバ側のCORS許可が必要
+        signal,
+        cache: 'no-cache', // 必要なら更新を強制
+    });
+
+    if (!res.ok) {
+        // ステータスコード付きで例外
+        throw new Error(`HTTP ${res.status} ${res.statusText} for ${url}`);
+    }
+
+    // 自動で JSON parse され、JSオブジェクトになる
+    return await res.json();
+}
+
+/**
+ * 指定のサーバ上パス (filePath) から JSON を取得し、
+ * ブラウザで downloadFileName という名前でダウンロードさせる。
+ *
+ * @param {string} downloadFileName - 例: "data.geojson" / "result.json"
+ * @param {string} filePath - 例: "/public/out/data.geojson" or "https://example.com/data.json"
+ * @param {object} [opts]
+ * @param {number} [opts.timeoutMs=15000] - タイムアウト(ms)
+ * @param {RequestInit} [opts.fetchInit] - fetch の追加オプション（ヘッダ/credentials等）
+ */
+async function downloadJSONFile(downloadFileName, filePath, opts = {}) {
+  const { timeoutMs = 15000, fetchInit = {} } = opts;
+
+  // タイムアウト制御
+  const ac = new AbortController();
+  const timer = setTimeout(() => ac.abort(new Error('timeout')), timeoutMs);
+
+  try {
+    const res = await fetch(filePath, {
+      method: 'GET',
+      headers: { 'Accept': 'application/json', ...(fetchInit.headers || {}) },
+      signal: ac.signal,
+      // CORS が必要なら credentials / mode などを fetchInit で渡す（例: { credentials: 'include' }）
+      ...fetchInit,
+    });
+
+    if (!res.ok) {
+      throw new Error(`HTTP ${res.status} ${res.statusText} for ${filePath}`);
+    }
+
+    // そのまま Blob 化（JSON で返ってくる想定）
+    // MIME が不正のときもこちらで application/json に寄せる
+    const blob = await res.blob();
+    const type = res.headers.get('content-type');
+    const jsonBlob = type && type.includes('application/json')
+      ? blob
+      : new Blob([await blob.arrayBuffer()], { type: 'application/json;charset=utf-8' });
+
+    const url = URL.createObjectURL(jsonBlob);
+    try {
+      const a = document.createElement('a');
+      a.href = url;
+      a.download = downloadFileName; // ダウンロード名を強制
+      document.body.appendChild(a);
+      a.click();
+      a.remove();
+    } finally {
+      // 生成したオブジェクトURLは必ず破棄
+      URL.revokeObjectURL(url);
+    }
+  } finally {
+    clearTimeout(timer);
+  }
+}

package/views/index.ejs

@@ -11,7 +11,9 @@
   <link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" crossorigin="anonymous" />
   <!-- Leaflet.draw CSS -->
   <link rel="stylesheet" href="https://unpkg.com/leaflet-draw@1.0.4/dist/leaflet.draw.css" crossorigin="anonymous" />
-  <link rel="stylesheet" href="/style.css" />
+  <link rel="stylesheet" href="style.css" />
+  <!-- Toastify -->
+  <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/toastify-js/src/toastify.min.css">
 </head>
 
 <body>
@@ -99,7 +101,8 @@
   <!-- Map to Image -->
   <script src="https://cdn.jsdelivr.net/npm/leaflet-easyprint@2.1.9/dist/bundle.min.js"></script>
   <link href="https://cdn.jsdelivr.net/npm/leaflet-easyprint@2.1.9/libs/leaflet.min.css" rel="stylesheet">
-
+  <!-- Toastify -->
+  <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/toastify-js"></script>
   <!-- Socket.IO JS -->
   <script src="https://cdn.socket.io/4.7.5/socket.io.min.js"></script>
   <!-- Visualize.js -->
@@ -145,15 +148,21 @@
       //console.log('Disconnected from server');
     });
 
-    let sessionId=null;
+    let sessionId = null;
     let visualizers = {};
-    let layerControl=null;
-    socket.on("welcome",async (res) => {
+    let results = {};
+    let files = {};
+    let layerControl = null;
+    socket.on("welcome", async (res) => {
       //console.log(`welcome ${res.sessionId} at ${new Date(res.time).toLocaleTimeString()}`);
-      console.log("VIS",res.visualizers);
+      //console.log("VIS", res.visualizers);
       sessionId = res.sessionId;
-      visualizers = Object.fromEntries(await Promise.all(res.visualizers.map(async vis => { const m = await import(new URL(`./visualizer/${vis}/web.js`, import.meta.url).href); return [vis, m.entry ?? m.default ?? m[vis]]; })));
-      console.log(visualizers);
+      visualizers = Object.fromEntries(await Promise.all(res.visualizers.map(async vis => {
+        const m = await import(new URL(`./visualizer/${vis}/web.js`,
+          import.meta.url).href);
+        return [vis, m.entry ?? m.default ?? m[vis]];
+      })));
+      //console.log(visualizers);
     });
 
     function generateLegend(legends) {
@@ -222,46 +231,124 @@
           crawled: acc.crawled + (g.crawled ?? 0),
           total: acc.total + (g.total ?? 0)
         }), {
-        crawled: 0,
-        total: 0
-      }
+          crawled: 0,
+          total: 0
+        }
       );
       if (total == 0) {
         crawled = 1, total = 1;
       }
-      const percent = Math.round((crawled / total) * 100);
-      console.log(`[Done] ${crawled} / ${total} --> ${percent}`);
+      const percent = Math.round((crawled / total) * 100000) / 1000;
+      //console.log(`[Done] ${crawled} / ${total} --> ${percent}`);
       setProgress(document.getElementById("progressCrawl"), percent);
       //ここでHexのプログレスグラデーションレイヤのOpacity調整
       highlightHexById(res.hexId, 1 - res.progress[res.hexId].percent, "#263238");
     });
 
     function isPlainObject(a) {
-      if (a === null || typeof a !== 'object') return false;  // 原始値/関数など除外
+      if (a === null || typeof a !== 'object') return false; // 原始値/関数など除外
       const proto = Object.getPrototypeOf(a);
-      return proto === Object.prototype || proto === null;     // {} / Object.create(null)
+      return proto === Object.prototype || proto === null; // {} / Object.create(null)
     }
+    socket.on("toast", (arg) => {
+      const option = {
+        text: "",
+        duration: arg.close ? -1 : 5000,
+        close: false,
+        gravity: 'bottom',
+        position: 'center',
+        className: "toast-" + arg.class,
+        ...arg
+      };
+      Toastify(option).showToast();
+      //https://apvarun.github.io/toastify-js/#
+    });
+    socket.on("result-file", async(arg, callback) => {
+      try {
+        const file_path = '/out/result.' + arg.resultId + '.json';
+        latestFile = file_path;
+        files[arg.resultId] = file_path;
+        console.log("file path",file_path);
+        results[arg.resultId] = await fetchJsonObject(file_path);
+        console.log('結果ファイル取得完了');
+      } catch (e) {
+        console.error('結果ファイル取得失敗:', e);
+      }
+      //console.log("ARK");
+      callback({
+        pong: Date.now(),
+        ok: true
+      }); // only one argument is expected
+    });
+    socket.on("result-chunk", (arg, callback) => {
+      if (arg.progress) {
+        setProgress(document.getElementById("progressCrawl"), Math.round((arg.progress.current / arg.progress.total) * 100000) / 1000);
+      }
+      results[arg.resultId] ??= {};
+      if (arg.kind === "primitive" || arg.kind === "null") {
+        //'number' | 'string' | 'boolean' | 'bigint' | 'symbol' | 'undefined'
+        //console.log(arg.path,arg.value);
+        switch (arg.type) {
+          case 'number':
+            setAt(results[arg.resultId], arg.path, Number(arg.value));
+            break;
+          case 'boolean':
+            setAt(results[arg.resultId], arg.path, arg.value);
+          case 'string':
+            setAt(results[arg.resultId], arg.path, String(arg.value));
+            break;
+          case 'bigint':
+            setAt(results[arg.resultId], arg.path, BigInt(arg.value));
+            break;
+          case 'symbol':
+            setAt(results[arg.resultId], arg.path, Symbol(arg.value));
+            break;
+          default:
+            setAt(results[arg.resultId], arg.path, null);
+        }
+      } else if (arg.kind === "object") {
+        setAt(results[arg.resultId], arg.path, {});
+      } else if (arg.kind === "array") {
+        setAt(results[arg.resultId], arg.path, []);
+      }
+      //console.log("ARK");
+      callback({
+        pong: Date.now(),
+        ok: true
+      }); // only one argument is expected
+    });
 
     let latestResult = {};
+    let latestFile = null;
     const overlays = {};
-    socket.on('result',async (res) => {
+    socket.on('result', async (res, callback) => {
+      //console.log("result");
       latestResult["_"]["visualizers"] = res.visualizers;
       latestResult["_"]["plugin"] = res.plugin;
       clearHexHighlight();
-      latestResult = {...latestResult, ...res.geoJson};
-      console.log("result");
-      console.log("geoJSON",res.geoJson);
-      for(const vis in visualizers){
-        const layers = await visualizers[vis](map, res.geoJson[vis], {palette:res.palette});
-        console.log("【レイヤ】",layers);
-        if((layers == null) ){
+      if (res.geoJson == null) {
+        res.geoJson = results[res.resultId];
+      } else {
+        results[res.resultId] = res.geoJson;
+      }
+      latestResult = {
+        ...latestResult,
+        ...res.geoJson
+      };
+      for (const vis in visualizers) {
+        const layers = await visualizers[vis](map, res.geoJson[vis], {
+          palette: res.palette
+        });
+        //console.log("【レイヤ】\n", JSON.stringify(layers,null,4));
+        if ((layers == null)) {
           //SKIP
-        }else if(layers.hasOwnProperty("type") && layers["type"]=="FeatureCollection"){
+        } else if (layers.hasOwnProperty("type") && layers["type"] == "FeatureCollection") {
           //レイヤ一つ
-          overlays[`[${vis}]`] =  layers;
-        }else{
-          for(const name in layers){
-            overlays[`[${vis}] ${name}`] =  layers[name];
+          overlays[`[${vis}]`] = layers;
+        } else {
+          for (const name in layers) {
+            console.log("Lay", vis, name);
+            overlays[`[${vis}] ${name}`] = layers[name];
           }
         }
       }
@@ -280,7 +367,7 @@
         });
       }
       ul.innerHTML = generateLegend(palette);
-      console.log(overlays);
+      //console.log(overlays);
       layerControl = L.control.layers([baseLayer], {
         ...overlays,
         "Hex Grid": hexLayer,
@@ -290,16 +377,24 @@
       });
       layerControl.addTo(map);
       document.getElementById("map_legend")?.appendChild(ul);
-      document.getElementById("download-json").addEventListener("click", () => {
+      document.getElementById("download-json").addEventListener("click", async () => {
         const stamp = new Date().toISOString().replace(/[:.]/g, "-");
-        downloadJSON(`data-${stamp}.json`, latestResult);
+        if(latestFile===null){
+          downloadJSON(`splatone-${stamp}.json`, latestResult);
+        }else{
+          await downloadJSONFile(`splatone-${stamp}.json`, latestFile);
+        }
         let sessionId = null;
       });
+      callback({
+        pong: Date.now(),
+        ok: true
+      });
     });
 
     // 地図
     const map = L.map('map', {
-      preferCanvas:true,
+      preferCanvas: true,
       zoomControl: true
     }).setView([lat, lon], 12);
     const baseLayer = L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', {
@@ -348,11 +443,11 @@
     L.easyPrint({
       title: 'マップの画像化とダウンロード',
       position: 'bottomright',
-      sizeModes: [ 'A4Landscape'],
+      sizeModes: ['A4Landscape'],
       exportOnly: true,
       filename: 'splatone_export'
     }).addTo(map);
-    
+
     // レイヤ参照
     let hexLayer = null;
     let triLayer = null;
@@ -603,7 +698,7 @@
       const cellSize = document.getElementById('cellSize').value.trim();
       const units = document.getElementById('units').value.trim();
       const keywords = document.getElementById('keywords').value.trim();
-      latestResult["_"]??={};
+      latestResult["_"] ??= {};
       latestResult["_"]["keywords"] = keywords;
       socket.once("hexgrid", (res) => {
         if (res.sessionId) sessionId = res.sessionId;
@@ -611,7 +706,7 @@
         const hex = res.hex;
         const triangles = res.triangles;
         latestResult["_"]["hex"] = hex;
-        latestResult["_"]["triangles"]=[triangles];
+        latestResult["_"]["triangles"] = [triangles];
         renderLayers(hex, triangles);
       });
       socket.emit("target", {

package/visualizer/bulky/node.js

@@ -1,6 +1,5 @@
 import { VisualizerBase } from '../../lib/VisualizerBase.js';
 import { featureCollection } from "@turf/turf";
-import { writeFileSync } from 'node:fs';
 
 export default class BulkyVisualizer extends VisualizerBase {
     static id = 'bulky';            // 一意ID（フォルダ名と一致させると運用しやすい）

package/visualizer/bulky/web.js

@@ -1,9 +1,11 @@
 let booted = false;
 export default async function main(map, geojson, options = {}) {
+    console.log("main");
     if (booted) return;
     booted = true;
     const layers = {};
     for (const cat in geojson) {
+        console.log(cat);
         const layer = addGeoJSONLayer(map, geojson[cat], {
             pointToLayer: (feature, latlng) => {
                 return L.circleMarker(latlng, {