pivotgrid-js 0.1.2 → 0.1.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pivotgrid-js",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Vanilla JS pivot table — no dependencies, no frameworks",
   "author": "Aleksandr Korolev <korolevalexa@gmail.com>",
   "license": "SEE LICENSE IN LICENSE",

package/providers/rest-provider.js

@@ -271,19 +271,29 @@ class RestProvider {
   // ── HTTP ───────────────────────────────────────────────────────────────────
 
   async _execute(query) {
-    const res = await fetch(this.url, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ query }),
-    });
+    let page = 0;
+    let allRows = [];
+    let hasMore = true;
+
+    while (hasMore) {
+      const res = await fetch(this.url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ query, page }),
+      });
+
+      if (!res.ok) {
+        const err = await res.json().catch(() => ({}));
+        throw new Error(`Server error ${res.status}: ${err.error || ''}`);
+      }
 
-    if (!res.ok) {
-      const err = await res.json().catch(() => ({}));
-      throw new Error(`Server error ${res.status}: ${err.error || ''}`);
+      const data = await res.json();
+      allRows = allRows.concat(data.rows);
+      hasMore = data.hasMore;
+      page++;
     }
 
-    const rows = await res.json();
-    return rows.map(row => {
+    return allRows.map(row => {
       const out = {};
       for (const k of Object.keys(row)) out[k.toLowerCase()] = row[k];
       return out;

package/server/README.md

@@ -42,7 +42,7 @@ All settings can also be configured via the Config Editor UI.
 
 | Method | Path | Description |
 |--------|------|-------------|
-| `POST` | `/query` | Execute a SELECT query, returns JSON array |
+| `POST` | `/query` | Execute a SELECT query, returns paginated JSON (`{rows, total, page, hasMore}`) |
 | `GET` | `/configs` | List all config names |
 | `GET` | `/configs/{name}` | Get a config by name |
 | `POST` | `/configs/{name}` | Save a config by name |
@@ -50,6 +50,30 @@ All settings can also be configured via the Config Editor UI.
 | `POST` | `/server-config` | Save DB settings to `.env` |
 | `POST` | `/test-connection` | Test DB connection with given credentials |
 
+## Query Pagination
+
+Large result sets are split into pages to avoid sending huge HTTP responses
+(which can fail or time out on slower connections).
+
+The query runs **once** — results are cached in memory on the server and
+sliced into pages as the client requests them, so a single `GROUP BY` is
+never re-executed for each page.
+
+```python
+_PAGE_SIZE = 200_000       # rows per page sent to the client
+_QUERY_CACHE_TTL = 300     # seconds a query's results stay in server memory
+```
+
+- `_PAGE_SIZE` — increase if your network/hardware comfortably handles
+  larger responses; decrease if you see slow or failing requests on big
+  datasets.
+- `_QUERY_CACHE_TTL` — how long the server keeps a query's full result in
+  memory while the client fetches subsequent pages. Stale entries are
+  cleaned up automatically.
+
+The client (`RestProvider`) handles pagination transparently — it requests
+pages in a loop and concatenates them into a single array.
+
 ## Configs
 
 Configs are stored as JSON files in `server/configs/`:

package/server/configs/main_config.json

@@ -106,7 +106,7 @@
   ],
   "measure": "revenue",
   "func": "sum",
-  "maxCachedRows": 500000,
+  "maxCachedRows": 1000000,
   "filterCheckboxLimit": 30,
   "drillthroughQuery": "\n    SELECT ID,REGION FROM sales_data\n    WHERE {filters}\n    LIMIT 200 OFFSET 0\n  "
 }

package/server/server.py

@@ -20,6 +20,8 @@ Start:
     python server.py
 """
 
+import time
+import hashlib
 import json
 import gzip
 import os
@@ -38,6 +40,18 @@ CONNECTORS_DIR  = os.path.join(BASE_DIR, 'connectors')
 
 os.makedirs(CONFIGS_DIR, exist_ok=True)
 
+_QUERY_CACHE = {} # query hash → {'rows': [...], 'ts': last access time}
+_QUERY_CACHE_TTL = 300 # seconds — how long a cached query result stays in memory
+_PAGE_SIZE = 200_000 # rows per page sent to the client in one HTTP response
+
+def _cache_key(query):
+    return hashlib.sha256(query.encode('utf-8')).hexdigest()
+
+def _cleanup_cache():
+    now = time.time()
+    for k in [k for k, v in _QUERY_CACHE.items() if now - v['ts'] > _QUERY_CACHE_TTL]:
+        del _QUERY_CACHE[k]
+
 # ── Load .env ──────────────────────────────────────────────────────────────────
 
 def load_env(path):
@@ -237,10 +251,27 @@ class Handler(BaseHTTPRequestHandler):
             self._send(400, json.dumps({'error': 'Only SELECT allowed'}))
             return
 
+        page = int(payload.get('page', 0))
+        key  = _cache_key(query)
+
         try:
-            connector = get_active_connector()
-            rows      = connector.execute_query(query)
-            self._send(200, json.dumps(rows, ensure_ascii=False, default=str))
+            if page == 0 or key not in _QUERY_CACHE:
+                _cleanup_cache()
+                connector = get_active_connector()
+                rows = connector.execute_query(query)
+                _QUERY_CACHE[key] = {'rows': rows, 'ts': time.time()}
+            else:
+                _QUERY_CACHE[key]['ts'] = time.time()
+
+            rows  = _QUERY_CACHE[key]['rows']
+            total = len(rows)
+            start = page * _PAGE_SIZE
+            chunk = rows[start:start + _PAGE_SIZE]
+            has_more = start + _PAGE_SIZE < total
+
+            self._send(200, json.dumps({
+                'rows': chunk, 'total': total, 'page': page, 'hasMore': has_more,
+            }, ensure_ascii=False, default=str))
         except Exception as e:
             self._send(500, json.dumps({'error': str(e)}))