splatone 0.0.8 → 0.0.9

package/README.md

@@ -12,6 +12,9 @@ SNSのジオタグ付きポストをキーワードに基づいて収集する
 - Marker Cluster: 密集しているジオタグをクラスタリングしてまとめて表示する
 
 ## Change Log
+### v0.0.8 →　v0.0.9 
+
+* クエリを時間方向でも分割し効率化しました。(使い方に変更はありません)
 
 ### v0.0.7 →　v0.0.8 
 

package/crawler.js

@@ -28,7 +28,7 @@ import { hideBin } from 'yargs/helpers';
 // -------------------------------
 import { loadPlugins } from './lib/pluginLoader.js';
 import paletteGenerator from './lib/paletteGenerator.js';
-import { dfsObject, bboxSize, saveGeoJsonObjectAsStream } from './lib/splatone.js';
+import { dfsObject, bboxSize, saveGeoJsonObjectAsStream, buildPluginsOptions, loadAPIKey } from '#lib/splatone';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -36,6 +36,7 @@ const VIZ_BASE = resolve(__dirname, "visualizer");
 const app = express();
 const port = 3000;
 const title = 'Splatone - Multi-Layer Composite Heatmap Viewer';
+let pluginsOptions = {};
 
 try {
 
@@ -164,6 +165,9 @@ try {
         catch (e) { throw new Error(`--${name}: JSON エラー: ${e.message}`); }
       })()
     });
+  plugins.list().forEach(async (plug) => {
+    yargv = await plugins.call(plug, "yargv", yargv);
+  })
   Object.keys(all_visualizers).forEach((vis) => {
     yargv = yargv.option('vis-' + vis, {
       group: 'Visualization (最低一つの指定が必須です)',
@@ -172,7 +176,7 @@ try {
       description: all_visualizers[vis].description
     })
   });
-  yargv = yargv.check((argv, options) => {
+  yargv = yargv.check(async (argv, options) => {
     if (Object.keys(all_visualizers).filter(v => argv["vis-" + v]).length == 0) {
       throw new Error('可視化ツールの指定がありません。最低一つは指定してください。');
     }
@@ -180,15 +184,18 @@ try {
       console.warn("--filedと--choppedが両方指定されています。--filedが優先されます。");
       argv.chopped = false;
     }
+    pluginsOptions = buildPluginsOptions(argv, plugins.list())
+    pluginsOptions[argv.plugin] = await plugins.call(argv.plugin, 'check', pluginsOptions[argv.plugin]);
     return true;
   });
   const argv = await yargv.parseAsync();
+
   const visualizers = {};
   for (const vis of Object.keys(all_visualizers).filter(v => argv[`vis-${v}`])) {
     visualizers[vis] = new all_visualizers[vis]();
   }
 
-  const plugin_options = argv.options?.[argv.plugin] ?? {}
+  /* const plugin_options = argv.options?.[argv.plugin] ?? {}
   try {
     plugin_options.API_KEY = await loadAPIKey("flickr") ?? plugin_options.API_KEY;
   } catch (e) {
@@ -196,48 +203,15 @@ try {
       console.error("Error loading API key:", e.message);
     }
     //Nothing to do
-  }
-  await plugins.call(argv.plugin, 'init', plugin_options);
+  }*/
+  //pluginsOptions = buildPluginsOptions(argv, plugins.list());
+  await plugins.call(argv.plugin, 'init', pluginsOptions[argv.plugin]);
   if (argv.debugVerbose) {
     console.table([["Visualizer", Object.keys(visualizers)], ["Plugin", argv.plugin]]);
   }
 
-  /* API Key読み込み */
-  async function loadAPIKey(plugin = 'flickr') {
-    //ファイルチェック→環境変数チェック
-
-    const filePath = ".API_KEY." + plugin;
-    const file = resolve(filePath);
-    // 存在＆読取権限チェック
-    let key = null;
-    try {
-      await access(file, constants.F_OK | constants.R_OK);
-      // 読み込み & トリム
-      const raw = await readFile(file, 'utf8');
-      //console.log(`[API KEY (${plugin}})] Read from FILE`);
-      key = raw.trim();
-    } catch (err) {
-      if (Object.prototype.hasOwnProperty.call(process.env, "API_KEY_" + plugin)) {
-        //console.log(`[API KEY (${plugin}})] Read from ENV`);
-        key = process.env["API_KEY_" + plugin] ?? null;
-      } else {
-        const code = /** @type {{ code?: string }} */(err).code || 'UNKNOWN';
-        throw new Error(`APIキーのファイルもしくは環境変数にアクセスできません: ${file} (code=${code})`);
-      }
-    }
-    if (!key) {
-      throw new Error(`APIキーのファイルが空です: ${file}`);
-    }
-    // ※任意: Flickr APIキーの緩い形式チェック（英数32+文字）
-    // 公式に厳格仕様が明示されていないため、緩めのガードに留めます。
-    if (!/^[A-Za-z0-9]{32,}$/.test(key)) {
-      // 形式が怪しい場合は警告だけにするなら console.warn に変更
-      throw new Error(`APIキーの形式が不正の可能性があります（英数字32文字以上を想定）: ${file}`);
-    }
-    return key;
-  }
-
 
+  const processing = {};
   const crawlers = {};
   const targets = {};
   // 初期中心（凱旋門）
@@ -267,6 +241,9 @@ try {
     return k1 < k2 ? k1 : k2;
   }
 
+  function unixTimeLocal(year, month, day, hour = 0, minute = 0, second = 0) {
+    return Math.round(new Date(year, month - 1, day, hour, minute, second).getTime() / 1000);
+  }
 
   function defaultMaxUploadTime(date = new Date()) {
     return Math.floor(date / 1000) - 360;
@@ -328,20 +305,21 @@ try {
     socket.emit("welcome", { socketId: socket.id, sessionId: sessionId, time: Date.now(), visualizers: Object.keys(visualizers) });
     //クローリング開始
     socket.on("crawling", async (req) => {
+
       try {
         if (sessionId !== req.sessionId) {
           console.warn("invalid sessionId:", req.sessionId);
           return;
         }
-
-        await plugins.call('flickr', 'crawl', {
+        const optPlugin = {
           hexGrid: targets[req.sessionId].hex,
           triangles: targets[req.sessionId].triangles,
           sessionId: req.sessionId,
-          //tags: targets[req.sessionId].tags,
           categories: targets[req.sessionId].categories,
-          max_upload_date: defaultMaxUploadTime(),
-        });
+          pluginOptions: pluginsOptions[argv.plugin]
+        };
+        //console.log(optPlugin);
+        await plugins.call(argv.plugin, 'crawl', optPlugin);
       }
       catch (e) {
         console.error(e);
@@ -352,7 +330,6 @@ try {
     // クロール範囲指定
     socket.on("target", (req) => {
       try {
-        //console.log("target:", req);
         if (sessionId !== req.sessionId) {
           console.warn("invalid sessionId:", req.sessionId);
           return;
@@ -365,11 +342,13 @@ try {
           const { width, height } = bboxSize(boundary, units);
           //console.log("","w=",width,"/\th=",height);
           cellSize = Math.max(width / (3 * 30), height / (30 * Math.sqrt(3)));
-          if(cellSize==0){
-            cellSize=1;
+          if (cellSize == 0) {
+            cellSize = 1;
           }
           const msg = "セルサイズを[ " + cellSize + ' ' + units + " ]に設定しました。";
-          console.log(msg)
+          if (argv.debugVerbose) {
+            console.log("INFO:", msg)
+          }
           io.to(sessionId).timeout(5000).emit('toast', {
             text: msg,
             class: "info"
@@ -510,7 +489,7 @@ try {
 
         //console.log(JSON.stringify(hexFC, null, 2));
         //res.json({ hex: hexFC, triangles: trianglesFC });
-        targets[sessionId] = { hex: hexFC, triangles: trianglesFC, categories, splatonePalette };
+        targets[sessionId] = { hex: hexFC, triangles: trianglesFC, categories, splatonePalette, };
         socket.emit("hexgrid", { hex: hexFC, triangles: trianglesFC });
       } catch (e) {
         console.error(e);
@@ -584,7 +563,9 @@ try {
   });
   await subscribe('splatone:start', async p => {
     //console.log('[splatone:start]', p);
-    const rtn = await runTask(p.plugin, p);
+    processing[p.sessionId] = (processing[p.sessionId] ?? 0) + 1;
+    let rtn = await runTask(p.plugin, p);
+    processing[p.sessionId]--;
     //console.log('[splatone:done]', p.plugin, rtn.photos.features.length,"photos are collected in hex",rtn.hexId,"tags:",rtn.tags,"final:",rtn.final);
     crawlers[p.sessionId][rtn.hexId] ??= {};
     crawlers[p.sessionId][rtn.hexId][rtn.category] ??= { items: featureCollection([]) };
@@ -595,20 +576,25 @@ try {
     crawlers[p.sessionId][rtn.hexId][rtn.category].crawled ??= 0;
     crawlers[p.sessionId][rtn.hexId][rtn.category].total = rtn.final ? crawlers[p.sessionId][rtn.hexId][rtn.category].ids.size : rtn.total + crawlers[p.sessionId][rtn.hexId][rtn.category].crawled;
     crawlers[p.sessionId][rtn.hexId][rtn.category].crawled = crawlers[p.sessionId][rtn.hexId][rtn.category].ids.size;
+    
     if (argv.debugVerbose) {
-      console.log(`(CRAWL) ${rtn.hexId} ${rtn.category} ] dup=${duplicates.size}, out=${rtn.outside}, in=${rtn.photos.features.length}  || ${crawlers[p.sessionId][rtn.hexId][rtn.category].crawled} / ${crawlers[p.sessionId][rtn.hexId][rtn.category].total}`);
+      console.log('INFO:', ` ${rtn.hexId} ${rtn.category} ] dup=${duplicates.size}, out=${rtn.outside}, in=${rtn.photos.features.length}  || ${crawlers[p.sessionId][rtn.hexId][rtn.category].crawled} / ${crawlers[p.sessionId][rtn.hexId][rtn.category].total}`);
     }
     const photos = featureCollection(rtn.photos.features.filter((f) => !duplicates.has(f.properties.id)));
     crawlers[p.sessionId][rtn.hexId][rtn.category].items
       = concatFC(crawlers[p.sessionId][rtn.hexId][rtn.category].items, photos);
+
     const { stats, progress, finish } = statsItems(crawlers[p.sessionId], targets[p.sessionId]);
     io.to(p.sessionId).emit('progress', { hexId: rtn.hexId, progress });
     if (!rtn.final) {
       // 次回クロール用に更新
-      p.max_upload_date = rtn.next_max_upload_date;
-      //console.log("next max_upload_date:", p.max_upload_date);
-      api.emit('splatone:start', p);
-    } else if (finish) {
+      rtn.nextPluginOptions.forEach((nextPluginOptions) => {
+        const p_clone = structuredClone(p);
+        p_clone.pluginOptions = nextPluginOptions
+        api.emit('splatone:start', p_clone);
+      });
+      //} else if (finish) {
+    } else if (processing[p.sessionId] == 0) {
       if (argv.debugVerbose) {
         console.table(stats);
       }
@@ -620,10 +606,9 @@ try {
     const resultId = uniqid();
     const result = crawlers[p.sessionId];
     const target = targets[p.sessionId];
-
     let geoJson = Object.fromEntries(Object.entries(visualizers).map(([vis, v]) => [vis, v.getFutureCollection(result, target)]));
 
-    console.log('[splatone:finish]');
+    //console.log('[splatone:finish]');
     try {
       if (argv.chopped || argv.filed) {
         throw new RangeError("Invalid string length");
@@ -723,6 +708,6 @@ try {
 
   process.on('SIGINT', async () => { await plugins.stopAll(); process.exit(0); });
 } catch (e) {
+  console.error("[SPLATONE ERROR]");
   console.error(e);
-
 }

package/lib/PluginBase.js

@@ -12,12 +12,27 @@ export class PluginBase {
     this.api = api;
     this.options = options;
   }
+
+  argKey(key) {
+    return "p-" + this.id + "-" + key;
+  }
+  async check(option) {
+    //throw Error("Plugin Option Error!");
+    return true;
+  }
+
   async init(options = {}) {
     Object.assign(this.options, options);
   }
+
+  async options(yargs) {
+
+    return yargs;
+  }
+
   async start() {
     this.started = true;
     //this.api.log(`[${this.constructor.id}] start`);
   }
-  async stop() {}
+  async stop() { }
 }

package/lib/pluginLoader.js

@@ -107,9 +107,9 @@ async function resolveEntry(folder) {
 function validateClass(PluginClass, file) {
   if (typeof PluginClass !== 'function') throw new Error(`Plugin must export a class: ${file}`);
   // 静的メタ
-  if (!PluginClass.id || typeof PluginClass.id !== 'string') {
+  /*if (!PluginClass.id || typeof PluginClass.id !== 'string') {
     throw new Error(`static id (string) is required: ${file}`);
-  }
+  }*/ //動的に決定する(ディレクトリ名)
   if (!PluginClass.version || typeof PluginClass.version !== 'string') {
     throw new Error(`static version (string) is required: ${file}`);
   }

package/lib/splatone.js

@@ -2,7 +2,7 @@
 // @turf/turf v6/v7 どちらでもOK（rhumbDistanceはv7で統合済み）
 import { point, distance, rhumbDistance, bbox as turfBbox } from '@turf/turf';
 import { createWriteStream } from 'node:fs';
-import { mkdir } from 'node:fs/promises';
+import { mkdir, constants, access, readFile } from 'node:fs/promises';
 import path from 'node:path';
 import { pipeline as pipelineCb } from 'node:stream';
 import { promisify } from 'node:util';
@@ -145,7 +145,94 @@ export async function saveGeoJsonObjectAsStream(geoJsonObject, outfile) {
   await pipeline(src, dest);
   return destPath;
 }
+
+export function buildPluginsOptions(argv, pluginIds) {
+  const out = {};
+  for (const id of pluginIds) {
+    const prefix = `p-${id}`;
+    const opts = {};
+    for (const [key, val] of Object.entries(argv)) {
+      if (key === '_' || key === '$0') continue;
+      if (key === prefix) {                 // --p-id=VALUE
+        opts.__value = val;
+        continue;
+      }
+      if (key.startsWith(prefix)) {         // --p-id.xxx or --p-id-xxx
+        const sep = key[prefix.length];
+        if (sep === '.' || sep === '-') {
+          let sub = key.slice(prefix.length + 1).replace(/-/g, '.');
+          if (sub) setDeep(opts, sub, val);
+        }
+      }
+    }
+    if (Object.keys(opts).length) out[id] = opts;
+  }
+  //return out;
+  const lowerRe = /^\p{Ll}/u;
+
+  const out2 = {};
+  for (const [k, v] of Object.entries(out)) {
+    if (v && typeof v === "object" && !Array.isArray(v)) {
+      // 子がプレーンオブジェクトの場合だけ二階層目をフィルタ
+      out2[k] = Object.fromEntries(
+        Object.entries(v).filter(([kk]) => kk !== "" && !lowerRe.test(kk))
+      );
+    } else {
+      // それ以外（配列・null・プリミティブ）はそのまま
+      out2[k] = v;
+    }
+  }
+  return out2;
+}
+
+function setDeep(obj, path, value) {
+  const segs = path.split('.');
+  let cur = obj;
+  for (let i = 0; i < segs.length - 1; i++) {
+    const k = segs[i];
+    cur = (cur[k] ??= {});
+  }
+  cur[segs[segs.length - 1]] = value;
+}
+
+/* API Key読み込み */
+export async function loadAPIKey(plugin = 'flickr') {
+  //ファイルチェック→環境変数チェック
+
+  const filePath = ".API_KEY." + plugin;
+  const file = path.resolve(filePath);
+  //console.log(file);
+  // 存在＆読取権限チェック
+  let key = null;
+  try {
+    await access(file, constants.F_OK | constants.R_OK);
+    // 読み込み & トリム
+    const raw = await readFile(file, 'utf8');
+    //console.log(`[API KEY (${plugin}})] Read from FILE`);
+    key = raw.trim();
+  } catch (err) {
+    if (Object.prototype.hasOwnProperty.call(process.env, "API_KEY_" + plugin)) {
+      //console.log(`[API KEY (${plugin}})] Read from ENV`);
+      key = process.env["API_KEY_" + plugin] ?? null;
+    } else {
+      const code = /** @type {{ code?: string }} */(err).code || 'UNKNOWN';
+      throw new Error(`APIキーのファイルもしくは環境変数にアクセスできません: ${file} (msg=${err.message})`);
+    }
+  }
+  if (!key) {
+    throw new Error(`APIキーのファイルが空です: ${file}`);
+  }
+  // ※任意: Flickr APIキーの緩い形式チェック（英数32+文字）
+  // 公式に厳格仕様が明示されていないため、緩めのガードに留めます。
+  if (!/^[A-Za-z0-9]{32,}$/.test(key)) {
+    // 形式が怪しい場合は警告だけにするなら console.warn に変更
+    throw new Error(`APIキーの形式が不正の可能性があります（英数字32文字以上を想定）: ${file}`);
+  }
+  return key;
+}
+
+/*
 export default {
-  dfsObject, bboxSize, saveGeoJsonObjectAsStream
-};
+  dfsObject, bboxSize, saveGeoJsonObjectAsStream, buildPluginsOptions, loadAPIKey
+};*/
 //const {width,height} = bboxSize(b, "kilometers", "geodesic");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "splatone",
-  "version": "0.0.8",
+  "version": "0.0.9",
   "description": "Multi-layer Composite Heatmap",
   "homepage": "https://github.com/YokoyamaLab/Splatone#readme",
   "bugs": {
@@ -17,6 +17,9 @@
   "bin": {
     "crawler": "./crawler.js"
   },
+  "imports": {
+    "#lib/*": "./lib/*.js"
+  },
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
   },

package/plugins/flickr/index.js

@@ -1,23 +1,102 @@
-// plugins/hello/index.js
+// plugins/flickr/index.js
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { PluginBase } from '../../lib/PluginBase.js';
 import { bbox, polygon, centroid, booleanPointInPolygon, featureCollection } from '@turf/turf';
-
+import { loadAPIKey } from '#lib/splatone';
 export default class FlickrPlugin extends PluginBase {
-    static id = 'flickr';               // 必須
-    static name = 'Flickr Plugin';      // 任意
+
+    static name = 'Flickr Plugin';       // 任意
     static description = 'Flickrからジオタグ付きデータを収集する。';
     static version = '1.0.0';
+    constructor(api, options = {}) {
+        super(api, options);
+        this.id = path.basename(path.dirname(fileURLToPath(import.meta.url)));//必須(ディレクトリ名がプラグイン名) 
+    }
+    async yargv(yargv) {
+        // 必須項目にすると、このプラグインを使用しない時も必須になります。
+        // 必須項目は作らず、initで例外を投げてください。
+        return yargv.option(this.argKey('APIKEY'), {
+            group: 'For ' + this.id + ' Plugin',
+            type: 'string',
+            description: 'Flickr ServiceのAPI KEY'
+        }).coerce(this.argKey('APIKEY'), opt => {
+            return opt
+        }).option(this.argKey('DateMax'), {
+            group: 'For ' + this.id + ' Plugin',
+            type: 'string',
+            default: Math.floor(new Date() / 1000) - 360,
+            description: 'クローリング期間(最大) UNIX TIMEもしくはYYYY-MM-DD'
+        }).coerce(this.argKey('DateMax'), opt => {
+            if (!opt) return opt;  // undefined/null はそのまま返す
+
+            // YYYY-MM-DD 形式のチェック
+            const dateMatch = /^(\d{4})-(\d{2})-(\d{2})$/.exec(opt);
+            if (dateMatch) {
+                const [_, year, month, day] = dateMatch;
+                const date = new Date(parseInt(year), parseInt(month) - 1, parseInt(day));
+                if (date.toString() === 'Invalid Date') {
+                    throw new Error(`Invalid date format: ${opt} (正しい日付を指定してください)`);
+                }
+                return Math.floor(date.getTime() / 1000);
+            }
+
+            // 数値文字列または数値のチェック
+            const num = Number(opt);
+            if (Number.isFinite(num)) {
+                return Math.floor(num);  // 確実に整数に
+            }
+
+            throw new Error(`Invalid date/time format: ${opt} (YYYY-MM-DD または UNIX時間(秒)で指定してください)`);
+        }).option(this.argKey('DateMin'), {
+            group: 'For ' + this.id + ' Plugin',
+            type: 'string',
+            default: Math.round(new Date(2004, 1 - 1, 1, 0, 0, 0).getTime() / 1000),
+            description: 'クローリング期間(最小) UNIX TIMEもしくはYYYY-MM-DD'
+        }).coerce(this.argKey('DateMin'), opt => {
+            if (!opt) return opt;  // undefined/null はそのまま返す
+
+            // YYYY-MM-DD 形式のチェック
+            const dateMatch = /^(\d{4})-(\d{2})-(\d{2})$/.exec(opt);
+            if (dateMatch) {
+                const [_, year, month, day] = dateMatch;
+                const date = new Date(parseInt(year), parseInt(month) - 1, parseInt(day));
+                if (date.toString() === 'Invalid Date') {
+                    throw new Error(`Invalid date format: ${opt} (正しい日付を指定してください)`);
+                }
+                return Math.floor(date.getTime() / 1000);
+            }
+
+            // 数値文字列または数値のチェック
+            const num = Number(opt);
+            if (Number.isFinite(num)) {
+                return Math.floor(num);  // 確実に整数に
+            }
+            throw new Error(`Invalid date/time format: ${opt} (YYYY-MM-DD または UNIX時間(秒)で指定してください)`)
+        });
+    }
+
+    async check(options) {
+        const RE_FLICKR_API_KEY = /^[0-9a-f]{32}$/i;
+        if (!options['APIKEY']) {
+            const apikey = await loadAPIKey(this.id);
+            //console.log(apikey);
+            options['APIKEY'] = apikey;
+        } else if (!RE_FLICKR_API_KEY.test(options['APIKEY'])) {
+            throw new Error('Invalid Flickr API key format: 32桁 16進数で指定してください');
+        }
+        return options;
+    }
 
     async stop() {
         //this.api.log(`[${this.constructor.id}] stop`);
     }
 
     // 任意の公開メソッド
-    async crawl({ hexGrid, triangles/*, tags*/, categories, max_upload_date, sessionId }) {
+    async crawl({ hexGrid, triangles/*, tags*/, categories, sessionId, pluginOptions }) {
         if (!this.started) {
             this.start();
         }
-
         const getTrianglesInHex = (hex, triangles) => {
             const hexPoly = polygon(hex.geometry.coordinates);
             const selected = triangles.features.filter(tri => {
@@ -38,18 +117,18 @@ export default class FlickrPlugin extends PluginBase {
                 //console.log("tag=",ck,"/",tags);
                 hexQuery[item.properties.hexId][ck] = { photos: [], tags, final: false };
                 this.api.emit('splatone:start', {
-                    plugin: 'flickr',
-                    API_KEY: this.options.API_KEY,
+                    plugin: this.id,
+                    //API_KEY: this.APIKEY ?? pluginOptions.APIKEY,
                     hex: item,
                     triangles: getTrianglesInHex(item, triangles),
                     bbox: bbox(item.geometry),
                     category: ck,
                     tags,
-                    max_upload_date,
+                    pluginOptions,
                     sessionId
                 });
             }));
         }));
-        return `Flickr, ${this.options.API_KEY}, ${hexGrid.features.length} bboxes processed.`;
+        return `${this.id}, ${this.options.API_KEY}, ${hexGrid.features.length} bboxes processed.`;
     }
 }

package/plugins/flickr/worker.js

@@ -3,21 +3,23 @@ import { point, featureCollection } from "@turf/helpers";
 import booleanPointInPolygon from "@turf/boolean-point-in-polygon";
 
 export default async function ({
-    API_KEY = "",
+    //API_KEY = "",
     bbox = [0, 0, 0, 0],
     tags = "",
     category = "",
-    max_upload_date = null,
     hex = null,
     triangles = null,
+    pluginOptions
 }) {
-    const { flickr } = createFlickr(API_KEY);
+    //console.log("{PLUGIN}", pluginOptions);
+    const { flickr } = createFlickr(pluginOptions["APIKEY"]);
     const baseParams = {
         bbox: bbox.join(','),
         tags: tags,
-        max_upload_date: max_upload_date,
-
+        max_upload_date: pluginOptions["DateMax"],
+        min_upload_date: pluginOptions["DateMin"],
     };
+    //console.log("[baseParams]",baseParams);
     const res = await flickr("flickr.photos.search", {
         ...baseParams,
         has_geo: 1,
@@ -27,7 +29,6 @@ export default async function ({
         sort: "date-posted-desc"
     });
     //console.log(baseParams);
-    //console.log("[(Crawl)", hex.properties.hexId, category, "]", (new Date(max_upload_date * 1000)).toLocaleString(), "-> photos:", res.photos.photo.length, "/", res.photos.total);
     const ids = [];
     const authors = {};
     const photos = featureCollection(res.photos.photo.filter(photo => {
@@ -54,7 +55,9 @@ export default async function ({
     }));
     const outside = res.photos.photo.length - photos.features.length;
     //console.log(JSON.stringify(photos, null, 4));
-    let next_max_upload_date
+
+    const nextPluginOptionsDelta = [];
+    let next_max_date
         = res.photos.photo.length > 0
             ? (res.photos.photo[res.photos.photo.length - 1].dateupload) - (res.photos.photo[res.photos.photo.length - 1].dateupload == res.photos.photo[0].dateupload ? 1 : 0)
             : null;
@@ -62,17 +65,47 @@ export default async function ({
     if (Object.keys(authors).length == 1 && window < 60 * 60) {
         const skip = window < 5 ? 0.1 : 12;
         console.warn("[Warning]", `High posting activity detected for ${Object.keys(authors)} within ${window} s. the crawler will skip the next ${skip} hours.`);
-        next_max_upload_date -= 60 * 60 * skip;
+        next_max_date -= 60 * 60 * skip;
+    }
+    if (res.photos.pages > 4) {
+        //結果の最大・最小を2分割
+        const mid = ((next_max_date - pluginOptions.DateMin) / 2) + pluginOptions.DateMin;
+        nextPluginOptionsDelta.push({
+            'DateMax': next_max_date,
+            'DateMin': mid
+        });
+        nextPluginOptionsDelta.push({
+            'DateMax': mid,
+            'DateMin': pluginOptions.DateMin
+        });
+    } else {
+        nextPluginOptionsDelta.push({
+            'DateMax': next_max_date,
+            'DateMin': pluginOptions.DateMin
+        });
     }
     return {
         photos,
         hexId: hex.properties.hexId,
         tags,
         category,
-        next_max_upload_date,
+        nextPluginOptions: nextPluginOptionsDelta.map(e => { return { ...pluginOptions, ...e } }),
         total: res.photos.total,
         outside: outside,
         ids,
         final: res.photos.photo.length == res.photos.total
     };
+    /*
+    pluginOptions["DateMax"] = next_max_date;
+    return {
+        photos,
+        hexId: hex.properties.hexId,
+        tags,
+        category,
+        nextPluginOptions: pluginOptions,
+        total: res.photos.total,
+        outside: outside,
+        ids,
+        final: res.photos.photo.length == res.photos.total
+    };*/
 }

package/views/index.ejs

@@ -347,7 +347,7 @@
           overlays[`[${vis}]`] = layers;
         } else {
           for (const name in layers) {
-            console.log("Lay", vis, name);
+            //console.log("Lay", vis, name);
             overlays[`[${vis}] ${name}`] = layers[name];
           }
         }

package/visualizer/bulky/web.js

@@ -1,11 +1,11 @@
 let booted = false;
 export default async function main(map, geojson, options = {}) {
-    console.log("main");
+    //console.log("main");
     if (booted) return;
     booted = true;
     const layers = {};
     for (const cat in geojson) {
-        console.log(cat);
+        //console.log(cat);
         const layer = addGeoJSONLayer(map, geojson[cat], {
             pointToLayer: (feature, latlng) => {
                 return L.circleMarker(latlng, {

package/visualizer/marker-cluster/web.js

@@ -125,7 +125,7 @@ export default async function main(map, geojson, options = { palette: {} }) {
 
         // まだマップに載っていないクラスターは載せる
         for (const [cat, grp] of clusterByCategory.entries()) {
-            console.log("add map", cat);
+            //console.log("add map", cat);
             if (!map.hasLayer(grp)) grp.addTo(map);
         }
 
@@ -157,7 +157,7 @@ export default async function main(map, geojson, options = { palette: {} }) {
 
     // 実データを投入（fetchで置き換え可）
     for (const cat in geojson) {
-        console.log(cat);
+        //console.log(cat);
         addFeatureCollectionByCategory(geojson[cat], cat);
     }
     // Mapから単なるObjectへ