splatone 0.0.11 → 0.0.12

package/README.md

@@ -13,6 +13,13 @@ SNSのジオタグ付きポストをキーワードに基づいて収集する
 
 ## Change Log
 
+### v0.0.11 →　v0.0.12
+
+* Bottleneckを導入しクエリ間隔を適正値に調整 (3 queries/ 3 sec.)
+* 時間軸分割並列処理のデフォルト化
+  * 地理的分割に加えて大量の結果がある場所は時間軸でもクエリを分解する
+  * 無効にするときは```--no-p-flickr-Haste```を付与
+
 ### v0.0.10 →　v0.0.11 
 
 * 時間軸として使用する日付を選択可能に (```--p-flickr-DateMode```)
@@ -23,6 +30,7 @@ SNSのジオタグ付きポストをキーワードに基づいて収集する
     * デフォルト値:　```date_upload,date_taken,owner_name,geo,url_s,tags```
       * これらはコマンドライン引数での指定の有無に関わらず付与されます
 * 自動指定時のHexGridの最小サイズを0.5kmに
+* [Bug Fix] 時間軸並列機能のバグ修正
 
 ### v0.0.8 →　v0.0.9 →　v0.0.10 
 
@@ -59,15 +67,16 @@ Debug
       --debug-verbose  デバッグ情報出力               [真偽] [デフォルト: false]
 
 For flickr Plugin
-      --p-flickr-APIKEY   Flickr ServiceのAPI KEY                       [文字列]
-      --p-flickr-extras   カンマ区切り/保持する写真のメタデータ(デフォルト値は記
-                          載の有無に関わらず保持)
+      --p-flickr-APIKEY    Flickr ServiceのAPI KEY                      [文字列]
+      --p-flickr-Extras    カンマ区切り/保持する写真のメタデータ(デフォルト値は
+                           記載の有無に関わらず保持)
        [文字列] [デフォルト: "date_upload,date_taken,owner_name,geo,url_s,tags"]
-      --p-flickr-Date     利用時間軸(update=Flickr投稿日時/taken=写真撮影日時)
+      --p-flickr-DateMode  利用時間軸(update=Flickr投稿日時/taken=写真撮影日時)
                     [選択してください: "upload", "taken"] [デフォルト: "upload"]
-      --p-flickr-DateMax  クローリング期間(最大) UNIX TIMEもしくはYYYY-MM-DD
-                                               [文字列] [デフォルト: 1762701683]
-      --p-flickr-DateMin  クローリング期間(最小) UNIX TIMEもしくはYYYY-MM-DD
+      --p-flickr-Haste     時間軸分割並列処理          [真偽] [デフォルト: true]
+      --p-flickr-DateMax   クローリング期間(最大) UNIX TIMEもしくはYYYY-MM-DD
+                                               [文字列] [デフォルト: 1762904780]
+      --p-flickr-DateMin   クローリング期間(最小) UNIX TIMEもしくはYYYY-MM-DD
                                                [文字列] [デフォルト: 1072882800]
 
 Visualization (最低一つの指定が必須です)

package/crawler.js

@@ -8,12 +8,14 @@ import os from 'node:os';
 import { EventEmitter } from 'node:events';
 import path, { resolve, dirname } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
-import { existsSync, writeFileSync, constants } from 'node:fs';
+import fs, { existsSync, writeFileSync, constants } from 'node:fs';
 import { access, readdir, readFile } from 'node:fs/promises';
+import { MessageChannel } from 'worker_threads';
 
 // -------------------------------
 // Third-party
 // -------------------------------
+import Bottleneck from 'bottleneck';
 import express from 'express';
 import open from 'open';
 import Piscina from 'piscina';
@@ -37,6 +39,10 @@ const app = express();
 const port = 3000;
 const title = 'Splatone - Multi-Layer Composite Heatmap Viewer';
 let pluginsOptions = {};
+const flickrLimiter = new Bottleneck({
+  maxConcurrent: 5,
+  minTime: 350, // 約3req/sec
+});
 
 try {
 
@@ -109,7 +115,11 @@ try {
   // コマンド例
   // node crawler.js -p flickr -o '{"flickr":{"API_KEY":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}' -k "商業=shop,souvenir,market,supermarket,pharmacy,store,department|食べ物=food,drink,restaurant,cafe,bar|美術 館=museum,art,exhibition,expo,sculpture,heritage|公園=park,garden,flower,green,pond,playground" --vis-bulky
   // node crawler.js -p flickr -k "水域=canal,channel,waterway,river,stream,watercourse,sea,ocean,gulf,bay,strait,lagoon,offshore|橋梁=bridge,overpass,flyover,aqueduct,trestle|通路=street,road,thoroughfare,roadway,avenue,boulevard,lane,alley,roadway,carriageway,highway,motorway|ランドマーク=church,sanctuary,chapel,cathedral,basilica,minster,abbey,temple,shrine" --vis-bulky
-  // node crawler.js -p flickr -k "水辺=sea,ocean,beach,river,delta,lake,coast,creek|緑地=forest,woods,turf,lawn,jungle,trees,rainforest,grove,savanna,steppe|砂漠=desert,dune,outback,barren,wasteland" --vis-bulky --filed
+
+  // node crawler.js -p flickr -k "水辺=sea,ocean,beach,river,delta,lake,coast,creek|緑地=forest,woods,turf,lawn,jungle,trees,rainforest,grove,savanna,steppe|砂漠=desert,dune,outback,barren,wasteland" --vis-bulky --filed --p-flickr-Haste --p-flickr-DateMode=taken
+
+  // node crawler.js -p flickr -k "商業=shop,souvenir,market,supermarket,pharmacy,drugstore,store,department,kiosk,bazaar,bookstore,cinema,showroom|飲食=bakery,food,drink,restaurant,cafe,bar,beer,wine,whiskey|文化施設=museum,gallery,theater,concert,library,monument,exhibition,expo,sculpture,heritage|公園=park,garden,flower,green,pond,playground" --vis-bulky --p-flickr-Hates --p-flickr-DateMode=taken
+
   let yargv = await yargs(hideBin(process.argv))
     .strict()                        // 未定義オプションはエラー
     .usage('使い方: $0 [options]')
@@ -184,7 +194,6 @@ try {
       console.warn("--filedと--choppedが両方指定されています。--filedが優先されます。");
       argv.chopped = false;
     }
-    console.log(argv);
     pluginsOptions = buildPluginsOptions(argv, plugins.list())
     pluginsOptions[argv.plugin] = await plugins.call(argv.plugin, 'check', pluginsOptions[argv.plugin]);
     return true;
@@ -313,7 +322,7 @@ try {
           console.warn("invalid sessionId:", req.sessionId);
           return;
         }
-        const optPlugin = {
+        const workerOptions = {
           hexGrid: targets[req.sessionId].hex,
           triangles: targets[req.sessionId].triangles,
           sessionId: req.sessionId,
@@ -321,7 +330,7 @@ try {
           pluginOptions: pluginsOptions[argv.plugin]
         };
         //console.log(optPlugin);
-        await plugins.call(argv.plugin, 'crawl', optPlugin);
+        await plugins.call(argv.plugin, 'crawl', workerOptions);
       }
       catch (e) {
         console.error(e);
@@ -551,64 +560,79 @@ try {
     return { stats, progress, finish: (finish == 0) };
   };
 
-  async function runTask(taskName, data) {
+  async function runTask_(taskName, data) {
+    const { port1, port2 } = new MessageChannel();
     const filename = resolveWorkerFilename(taskName); // ← file URL (href)
     // named export を呼ぶ場合は { name: "関数名" } を追加
-    return piscina.run(data, { filename });
+    port1.on('message', (workerResults) => {
+      // ここでログ／WebSocket通知／DB書き込みなど何でもOK
+      const rtn = workerResults.results;
+      const workerOptions = workerResults.workerOptions;
+      const currentSessionId = workerOptions.sessionId;
+      processing[currentSessionId]--;
+      //console.log(rtn);
+      crawlers[currentSessionId][rtn.hexId] ??= {};
+      crawlers[currentSessionId][rtn.hexId][rtn.category] ??= { items: featureCollection([]) };
+      crawlers[currentSessionId][rtn.hexId][rtn.category].ids ??= new Set();
+      const duplicates = ((A, B) => new Set([...A].filter(x => B.has(x))))(rtn.ids, crawlers[currentSessionId][rtn.hexId][rtn.category].ids);
+      crawlers[currentSessionId][rtn.hexId][rtn.category].ids = new Set([...crawlers[currentSessionId][rtn.hexId][rtn.category].ids, ...rtn.ids]);
+      crawlers[currentSessionId][rtn.hexId][rtn.category].final = rtn.final;
+      crawlers[currentSessionId][rtn.hexId][rtn.category].crawled ??= 0;
+      crawlers[currentSessionId][rtn.hexId][rtn.category].total = rtn.final ? crawlers[currentSessionId][rtn.hexId][rtn.category].ids.size : rtn.total + crawlers[currentSessionId][rtn.hexId][rtn.category].crawled;
+      crawlers[currentSessionId][rtn.hexId][rtn.category].crawled = crawlers[currentSessionId][rtn.hexId][rtn.category].ids.size;
+
+      if (rtn.photos.features.length >= 250 && rtn.photos.features.length == duplicates.size) {
+        console.error("ALL DUPLICATE");
+      }
+      if (argv.debugVerbose) {
+        console.log('INFO:', ` ${rtn.hexId} ${rtn.category} ] dup=${duplicates.size}, out=${rtn.outside}, in=${rtn.photos.features.length}  || ${crawlers[currentSessionId][rtn.hexId][rtn.category].crawled} / ${crawlers[currentSessionId][rtn.hexId][rtn.category].total}`);
+      }
+      const photos = featureCollection(rtn.photos.features.filter((f) => !duplicates.has(f.properties.id)));
+      crawlers[currentSessionId][rtn.hexId][rtn.category].items
+        = concatFC(crawlers[currentSessionId][rtn.hexId][rtn.category].items, photos);
+
+      const { stats, progress, finish } = statsItems(crawlers[currentSessionId], targets[currentSessionId]);
+      io.to(currentSessionId).emit('progress', { hexId: rtn.hexId, progress });
+      if (!rtn.final) {
+        // 次回クロール用に更新
+        rtn.nextPluginOptions.forEach((nextPluginOptions) => {
+          const workerOptions_clone = structuredClone(workerOptions);
+          workerOptions_clone.pluginOptions = nextPluginOptions;
+          api.emit('splatone:start', workerOptions_clone);
+        });
+        //} else if (finish) {
+      } else if (processing[currentSessionId] == 0) {
+        if (argv.debugVerbose) {
+          console.table(stats);
+        }
+        api.emit('splatone:finish', workerOptions);
+      }
+    });
+    const rtn = await piscina.run({ debugVerbose: argv.debugVerbose, port: port2, ...data }, { filename, transferList: [port2] });
+    port1.close();
+    return rtn;
   }
+  const runTask = flickrLimiter.wrap(runTask_);
 
-  const nParallel = Math.max(1, Math.min(6, os.cpus().length))
+  const nParallel = Math.max(1, Math.min(12, os.cpus().length))
   const piscina = new Piscina({
-    minThreads: 1,
+    minThreads: nParallel,
     maxThreads: nParallel,
-    idleTimeout: 10_000,
-    // 注意：ここで filename は渡さない。run 時に切り替える
+    idleTimeout: 10_000
   });
-  await subscribe('splatone:start', async p => {
-    //console.log('[splatone:start]', p);
-    processing[p.sessionId] = (processing[p.sessionId] ?? 0) + 1;
-    let rtn = await runTask(p.plugin, p);
-    processing[p.sessionId]--;
-    //console.log('[splatone:done]', p.plugin, rtn.photos.features.length,"photos are collected in hex",rtn.hexId,"tags:",rtn.tags,"final:",rtn.final);
-    crawlers[p.sessionId][rtn.hexId] ??= {};
-    crawlers[p.sessionId][rtn.hexId][rtn.category] ??= { items: featureCollection([]) };
-    crawlers[p.sessionId][rtn.hexId][rtn.category].ids ??= new Set();
-    const duplicates = ((A, B) => new Set([...A].filter(x => B.has(x))))(rtn.ids, crawlers[p.sessionId][rtn.hexId][rtn.category].ids);
-    crawlers[p.sessionId][rtn.hexId][rtn.category].ids = new Set([...crawlers[p.sessionId][rtn.hexId][rtn.category].ids, ...rtn.ids]);
-    crawlers[p.sessionId][rtn.hexId][rtn.category].final = rtn.final;
-    crawlers[p.sessionId][rtn.hexId][rtn.category].crawled ??= 0;
-    crawlers[p.sessionId][rtn.hexId][rtn.category].total = rtn.final ? crawlers[p.sessionId][rtn.hexId][rtn.category].ids.size : rtn.total + crawlers[p.sessionId][rtn.hexId][rtn.category].crawled;
-    crawlers[p.sessionId][rtn.hexId][rtn.category].crawled = crawlers[p.sessionId][rtn.hexId][rtn.category].ids.size;
-
-    if (argv.debugVerbose) {
-      console.log('INFO:', ` ${rtn.hexId} ${rtn.category} ] dup=${duplicates.size}, out=${rtn.outside}, in=${rtn.photos.features.length}  || ${crawlers[p.sessionId][rtn.hexId][rtn.category].crawled} / ${crawlers[p.sessionId][rtn.hexId][rtn.category].total}`);
-    }
-    const photos = featureCollection(rtn.photos.features.filter((f) => !duplicates.has(f.properties.id)));
-    crawlers[p.sessionId][rtn.hexId][rtn.category].items
-      = concatFC(crawlers[p.sessionId][rtn.hexId][rtn.category].items, photos);
-
-    const { stats, progress, finish } = statsItems(crawlers[p.sessionId], targets[p.sessionId]);
-    io.to(p.sessionId).emit('progress', { hexId: rtn.hexId, progress });
-    if (!rtn.final) {
-      // 次回クロール用に更新
-      rtn.nextPluginOptions.forEach((nextPluginOptions) => {
-        const p_clone = structuredClone(p);
-        p_clone.pluginOptions = nextPluginOptions
-        api.emit('splatone:start', p_clone);
-      });
-      //} else if (finish) {
-    } else if (processing[p.sessionId] == 0) {
-      if (argv.debugVerbose) {
-        console.table(stats);
-      }
-      api.emit('splatone:finish', p);
-    }
+
+  await subscribe('splatone:start', async workerOptions => {
+    //console.log('[splatone:start]', workerOptions);
+    const currentSessionId = workerOptions.sessionId;
+    processing[currentSessionId] = (processing[currentSessionId] ?? 0) + 1;
+    runTask(workerOptions.plugin, workerOptions);
   });
 
-  await subscribe('splatone:finish', async p => {
+  await subscribe('splatone:finish', async workerOptions => {
+    const currentSessionId = workerOptions.sessionId;
     const resultId = uniqid();
-    const result = crawlers[p.sessionId];
-    const target = targets[p.sessionId];
+    const result = crawlers[currentSessionId];
+    const target = targets[currentSessionId];
     let geoJson = Object.fromEntries(Object.entries(visualizers).map(([vis, v]) => [vis, v.getFutureCollection(result, target)]));
 
     //console.log('[splatone:finish]');
@@ -616,7 +640,7 @@ try {
       if (argv.chopped || argv.filed) {
         throw new RangeError("Invalid string length");
       }
-      await io.to(p.sessionId).timeout(120000).emitWithAck('result', {
+      await io.to(currentSessionId).timeout(120000).emitWithAck('result', {
         resultId,
         geoJson,
         palette: target["splatonePalette"],
@@ -629,7 +653,7 @@ try {
         if (argv.debugVerbose) {
           console.warn("[WARN] " + msg);
         }
-        io.to(p.sessionId).timeout(5000).emit('toast', {
+        io.to(currentSessionId).timeout(5000).emit('toast', {
           text: msg,
           class: "warning"
         });
@@ -641,7 +665,7 @@ try {
             if (path.length !== 0) {
               if (kind === "primitive" || kind === "null") {
                 //console.log(path.join("."), "=>", `(${kind}:${type})`, value);
-                const ackrtn = await io.to(p.sessionId).timeout(120000).emitWithAck('result-chunk', {
+                const ackrtn = await io.to(currentSessionId).timeout(120000).emitWithAck('result-chunk', {
                   resultId,
                   path,
                   kind,
@@ -654,7 +678,7 @@ try {
                 if (path.at(-2) == "features" && Number.isInteger(path.at(-1))) {
                   current_features++;
                 }
-                const ackrtn = await io.to(p.sessionId).timeout(120000).emitWithAck('result-chunk', {
+                const ackrtn = await io.to(currentSessionId).timeout(120000).emitWithAck('result-chunk', {
                   resultId,
                   path,
                   kind,
@@ -664,7 +688,7 @@ try {
                 //console.log("\tACK", ackrtn);
               } else if (kind === "array") {
                 //console.log(path.join("."), "=>", `(${kind}:${type})`);              
-                const ackrtn = await io.to(p.sessionId).timeout(120000).emitWithAck('result-chunk', {
+                const ackrtn = await io.to(currentSessionId).timeout(120000).emitWithAck('result-chunk', {
                   resultId,
                   path,
                   kind,
@@ -682,7 +706,7 @@ try {
           try {
             const outPath = await saveGeoJsonObjectAsStream(geoJson, 'result.' + resultId + '.json');
             console.log('saved:', outPath);
-            const ackrtn = await io.to(p.sessionId).timeout(120000).emitWithAck('result-file', {
+            const ackrtn = await io.to(currentSessionId).timeout(120000).emitWithAck('result-file', {
               resultId,
             });
           } catch (err) {
@@ -690,7 +714,7 @@ try {
             process.exitCode = 1;
           }
         }
-        await io.to(p.sessionId).timeout(120000).emitWithAck('result', {
+        await io.to(currentSessionId).timeout(120000).emitWithAck('result', {
           resultId,
           geoJson: null, /*geoJsonは送らない*/
           palette: target["splatonePalette"],

package/lib/splatone.js

@@ -4,8 +4,8 @@ import { point, distance, rhumbDistance, bbox as turfBbox } from '@turf/turf';
 import { createWriteStream } from 'node:fs';
 import { mkdir, constants, access, readFile } from 'node:fs/promises';
 import path from 'node:path';
-import { pipeline as pipelineCb } from 'node:stream';
-import { promisify } from 'node:util';
+import { pipeline } from 'node:stream/promises';
+//import { promisify } from 'node:util';
 import { JsonStreamStringify } from 'json-stream-stringify';
 
 /**
@@ -45,6 +45,21 @@ export function bboxSizeOf(geojson, units = "kilometers", method = "geodesic") {
   return bboxSize(turfBbox(geojson), units, method);
 }
 
+/**
+ * 
+ * @param {*} dateTimeStr 日付文字列
+ * @returns UNIX Time Stamp (sec.)
+ */
+
+export function toUnixSeconds(dateTimeStr) {
+  const [datePart, timePart] = dateTimeStr.split(' ');
+  const [year, month, day] = datePart.split('-').map(Number);
+  const [hour, minute, second] = timePart.split(':').map(Number);
+
+  // JSの month は 0 始まり
+  const d = new Date(year, month - 1, day, hour, minute, second);
+  return Math.floor(d.getTime() / 1000);
+}
 
 // visit({ key, value, path, parent, depth, kind, type, isLeaf, isNull })
 //   kind: 'array' | 'object' | 'primitive' | 'null'
@@ -133,7 +148,6 @@ function describeType(value) {
   const ctor = value?.constructor?.name ?? "Object";
   return { kind: "object", type: ctor, isLeaf: false, isNull: false };
 }
-const pipeline = promisify(pipelineCb);
 export async function saveGeoJsonObjectAsStream(geoJsonObject, outfile) {
   const dir = path.join('public', 'out');
   await mkdir(dir, { recursive: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "splatone",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "description": "Multi-layer Composite Heatmap",
   "homepage": "https://github.com/YokoyamaLab/Splatone#readme",
   "bugs": {
@@ -25,6 +25,7 @@
   },
   "dependencies": {
     "@turf/turf": "^7.2.0",
+    "bottleneck": "^2.19.5",
     "chroma-js": "^3.1.2",
     "ejs": "^3.1.10",
     "express": "^5.1.0",

package/plugins/flickr/index.js

@@ -15,7 +15,7 @@ export default class FlickrPlugin extends PluginBase {
     }
     async yargv(yargv) {
         // 必須項目にすると、このプラグインを使用しない時も必須になります。
-        // 必須項目は作らず、initで例外を投げてください。
+        // 必須項目は作らず、もしプラグインを使う上での制約違反はinitで例外を投げてください。
         return yargv.option(this.argKey('APIKEY'), {
             group: 'For ' + this.id + ' Plugin',
             type: 'string',
@@ -45,9 +45,9 @@ export default class FlickrPlugin extends PluginBase {
             description: '利用時間軸(update=Flickr投稿日時/taken=写真撮影日時)'
         }).option(this.argKey('Haste'), {
             group: 'For ' + this.id + ' Plugin',
-            default: false,
+            default: true,
             type: 'boolean',
-            description: '時間軸分割並列処理(実験中の機能)'
+            description: '時間軸分割並列処理'
         }).option(this.argKey('DateMax'), {
             group: 'For ' + this.id + ' Plugin',
             type: 'string',
@@ -133,17 +133,16 @@ export default class FlickrPlugin extends PluginBase {
         }
         const hexQuery = {};
         const ks = Object.keys(hexGrid.features);
-        await Promise.all(ks.map(async k => {
+        ks.map(k => {
             const item = hexGrid.features[k];
             hexQuery[item.properties.hexId] = {};
             const cks = Object.keys(categories);
-            await Promise.all(cks.map(ck => {
+            cks.map(ck => {
                 const tags = categories[ck];
                 //console.log("tag=",ck,"/",tags);
                 hexQuery[item.properties.hexId][ck] = { photos: [], tags, final: false };
-                this.api.emit('splatone:start', {
+                this.api.emit('splatone:start', { //WorkerOptions
                     plugin: this.id,
-                    //API_KEY: this.APIKEY ?? pluginOptions.APIKEY,
                     hex: item,
                     triangles: getTrianglesInHex(item, triangles),
                     bbox: bbox(item.geometry),
@@ -152,8 +151,8 @@ export default class FlickrPlugin extends PluginBase {
                     pluginOptions,
                     sessionId
                 });
-            }));
-        }));
+            });
+        });
         return `${this.id}, ${this.options.API_KEY}, ${hexGrid.features.length} bboxes processed.`;
     }
 }

package/plugins/flickr/worker.js

@@ -1,26 +1,35 @@
 import { createFlickr } from "flickr-sdk"
 import { point, featureCollection } from "@turf/helpers";
 import booleanPointInPolygon from "@turf/boolean-point-in-polygon";
+import { toUnixSeconds } from '#lib/splatone';
 
 export default async function ({
-    //API_KEY = "",
-    bbox = [0, 0, 0, 0],
-    tags = "",
-    category = "",
-    hex = null,
-    triangles = null,
-    pluginOptions
+    port,
+    debugVerbose,
+    plugin,
+    hex,
+    triangles,
+    bbox,
+    category,
+    tags,
+    pluginOptions,
+    sessionId
 }) {
+    debugVerbose = true;
     //console.log("{PLUGIN}", pluginOptions);
     const { flickr } = createFlickr(pluginOptions["APIKEY"]);
+    if (!pluginOptions.TermId) {
+        //初期TermId
+        pluginOptions.TermId = 'a';
+    }
     const baseParams = {
         bbox: bbox.join(','),
         tags: tags,
         extras: pluginOptions["Extras"],
         sort: pluginOptions["DateMode"] == "upload" ? "date-posted-desc" : "date-taken-desc"
     };
-    baseParams[pluginOptions["Date"] == "upload" ? 'max_upload_date' : 'max_taken_date'] = pluginOptions["DateMax"];
-    baseParams[pluginOptions["Date"] == "upload" ? 'min_upload_date' : 'min_taken_date'] = pluginOptions["DateMin"];
+    baseParams[pluginOptions["DateMode"] == "upload" ? 'max_upload_date' : 'max_taken_date'] = pluginOptions["DateMax"];
+    baseParams[pluginOptions["DateMode"] == "upload" ? 'min_upload_date' : 'min_taken_date'] = pluginOptions["DateMin"];
     //console.log("[baseParams]",baseParams);
     const res = await flickr("flickr.photos.search", {
         ...baseParams,
@@ -28,7 +37,7 @@ export default async function ({
         per_page: 250,
         page: 1,
     });
-    //console.log(baseParams);
+    //console.log(res);
     const ids = [];
     const authors = {};
     const photos = featureCollection(res.photos.photo.filter(photo => {
@@ -57,33 +66,91 @@ export default async function ({
     //console.log(JSON.stringify(photos, null, 4));
 
     const nextPluginOptionsDelta = [];
-    let next_max_date
-        = res.photos.photo.length > 0
-            ? (res.photos.photo[res.photos.photo.length - 1].dateupload) - (res.photos.photo[res.photos.photo.length - 1].dateupload == res.photos.photo[0].dateupload ? 1 : 0)
-            : null;
-    const window = res.photos.photo.length == 0 ? 0 : res.photos.photo[0].dateupload - res.photos.photo[res.photos.photo.length - 1].dateupload;
-    if (Object.keys(authors).length == 1 && window < 60 * 60) {
-        const skip = window < 5 ? 0.1 : 12;
-        console.warn("[Warning]", `High posting activity detected for ${Object.keys(authors)} within ${window} s. the crawler will skip the next ${skip} hours.`);
-        next_max_date -= 60 * 60 * skip;
-    }
-    if (pluginOptions["Haste"] && res.photos.pages > 4) {
-        //結果の最大・最小を2分割
-        const mid = ((next_max_date - pluginOptions.DateMin) / 2) + pluginOptions.DateMin;
-        nextPluginOptionsDelta.push({
-            'DateMax': next_max_date,
-            'DateMin': mid
-        });
-        nextPluginOptionsDelta.push({
-            'DateMax': mid,
-            'DateMin': pluginOptions.DateMin
-        });
+    if (res.photos.photo.length == 0) {
+        if (debugVerbose) {
+            console.log(`Zero (${hex.properties.hexId} - ${category} - ${pluginOptions.TermId})`);
+        }
     } else {
-        nextPluginOptionsDelta.push({
-            'DateMax': next_max_date,
-            'DateMin': pluginOptions.DateMin,
-        });
+        let minDate, maxDate;
+        try {
+            minDate = res.photos.photo[res.photos.photo.length - 1].dateupload;
+            maxDate = res.photos.photo[0].dateupload
+
+            if (pluginOptions["DateMode"] == "taken") {
+                minDate = toUnixSeconds(res.photos.photo[res.photos.photo.length - 1].datetaken)
+                maxDate = toUnixSeconds(res.photos.photo[0].datetaken);
+            }
+        } catch {
+            console.log("DateMode ERROR")
+            console.log(res)
+        }
+        let next_max_date
+            = res.photos.photo.length > 0
+                ? (minDate) - (minDate == maxDate ? 1 : 0)
+                : null;
+        const window = res.photos.photo.length == 0 ? 0 : maxDate - minDate;
+        if (Object.keys(authors).length == 1 && res.photos.photo.length >= 250 && window < 60 * 60) {
+            const skip = window < 0 ? Math.abs(window) * 1.1 : (window < 5 ? 0.1 : 12);
+            if (debugVerbose) {
+                console.warn("[Warning]", (window < 0 ? "[[[Negative Time Window Error]]]" : ""), `High posting activity detected for ${Object.keys(authors)} within ${window} s. the crawler will skip the next ${skip} hours.`);
+            }
+            next_max_date -= 60 * 60 * skip;
+        }
+        if (pluginOptions["Haste"] && res.photos.pages > 4) {
+            //結果の最大・最小を2分割
+            const mid = Math.round(((next_max_date - pluginOptions.DateMin) / 2) + pluginOptions.DateMin);
+            if (debugVerbose) {
+                console.log(`Split(${hex.properties.hexId} - ${category} - ${pluginOptions.TermId}):`, pluginOptions.DateMin, mid, next_max_date);
+            }
+            nextPluginOptionsDelta.push({
+                'DateMax': next_max_date,
+                'DateMin': mid,
+                'TermId': pluginOptions.TermId + 'a'
+            });
+            nextPluginOptionsDelta.push({
+                'DateMax': mid,
+                'DateMin': pluginOptions.DateMin,
+                'TermId': pluginOptions.TermId + 'b'
+            });
+        } else if (res.photos.photo.length < res.photos.total) {
+            if (debugVerbose) {
+                console.log(`Continue[${res.photos.pages} pages](${hex.properties.hexId} - ${category} - ${pluginOptions.TermId}):`, pluginOptions.DateMin, next_max_date);
+            }
+            nextPluginOptionsDelta.push({
+                'DateMax': next_max_date,
+                'DateMin': pluginOptions.DateMin,
+            });
+        } else {
+            //final
+            if (debugVerbose) {
+                console.log(`Final(${hex.properties.hexId} - ${category} - ${pluginOptions.TermId}):`, pluginOptions.DateMin, next_max_date);
+            }
+        }
     }
+    port.postMessage({
+        workerOptions: {
+            plugin,
+            hex,
+            triangles,
+            bbox,
+            category,
+            tags,
+            pluginOptions,
+            sessionId,
+        },
+        results: {
+            photos,
+            hexId: hex.properties.hexId,
+            tags,
+            category,
+            nextPluginOptions: nextPluginOptionsDelta.map(e => { return { ...pluginOptions, ...e } }),
+            total: res.photos.total,
+            outside: outside,
+            ids,
+            final: nextPluginOptionsDelta.length == 0//res.photos.photo.length == res.photos.total
+        }
+    });
+    return true;
     return {
         photos,
         hexId: hex.properties.hexId,