pivotgrid-js 0.1.0

package/providers/rest-provider.js

@@ -0,0 +1,296 @@
+/**
+ * RestProvider
+ *
+ * Caching strategy:
+ *   - On startup, one GROUP BY over cachedDimensions → cache (ColumnStore)
+ *   - Everything else is lazy, not cached
+ *   - countRows(dims) → COUNT query for UI validation before adding to cache
+ *   - refreshCache(dims) → clear cache + new GROUP BY
+ */
+class RestProvider {
+
+  constructor({ url, query, dimensions, measures, funcs, fields = {},
+    cachedDimensions = [], maxCachedRows = 500_000, drillthroughQuery = null }) {
+    this.url = url;
+    this.query = query;
+    this.dimensions = dimensions;
+    this.measures = measures;
+    this.funcs = funcs;
+    this.fields = fields;
+    this.maxCachedRows = maxCachedRows;
+
+    this._cachedDims = [...cachedDimensions];
+    this._store = null;  // single ColumnStore cache
+    this._cacheRows = 0;     // rows in cache after last prefetch
+
+    this.drillthroughQuery = drillthroughQuery;
+  }
+
+  // ── Public API ─────────────────────────────────────────────────────────────
+
+  /**
+   * Initial load: one GROUP BY over cachedDimensions.
+   * Does nothing if the list is empty.
+   */
+  async prefetch() {
+    this._store = null;
+    this._cacheRows = 0;
+
+    if (!this._cachedDims.length) return;
+
+    const rows = await this._fetchGroupBy(this._cachedDims);
+    this._store = this._makeStore(this._cachedDims, rows);
+    this._cacheRows = rows.length;
+  }
+
+  /**
+   * COUNT of rows for a GROUP BY over the given set of dimensions.
+   * Used by CacheManager to validate before adding a dimension to cache.
+   * @param {string[]} logicalFields — logical field names (from CONFIG.dimensions)
+   * @returns {Promise<number>}
+   */
+  async countRows(logicalFields) {
+    if (!logicalFields.length) return 0;
+    const cols = this._expandFields(logicalFields).join(', ');
+    const sql = `
+      SELECT COUNT(*) AS cnt
+      FROM (
+        SELECT ${cols}
+        FROM   (${this.query}) _t
+        GROUP  BY ${cols}
+      ) _g
+    `;
+    const rows = await this._execute(sql);
+    return Number(rows[0]?.cnt || 0);
+  }
+
+  /**
+   * Clears the cache and reloads GROUP BY over the new set of dimensions.
+   * Called by the "Refresh cache" button.
+   */
+  async refreshCache(newDims) {
+    this._cachedDims = [...newDims];
+    await this.prefetch();
+  }
+
+  /** Current list of cached dimensions. */
+  get cachedDimensions() { return [...this._cachedDims]; }
+
+  /** Number of rows in cache (updated after prefetch/refreshCache). */
+  get cacheRows() { return this._cacheRows; }
+
+  // ── Grid data ────────────────────────────────────────────────────────────────
+
+  /**
+   * Returns iterable rows from cache if the cache covers requiredDims.
+   * Otherwise returns null.
+   */
+  getBestRows(requiredDims = [], activeFilters = {}) {
+    if (!this._store) return null;
+
+    const hasAllRequired = requiredDims.every(dim => {
+      const col = (this.fields[dim] || {}).label || dim;
+      return this._store.dimensions.includes(col);
+    });
+    if (!hasAllRequired) return null;
+
+    // If any filter dimension is missing from store — fall back to lazy SQL with WHERE
+    const filterDims = Object.keys(activeFilters);
+    const hasAllFilterDims = filterDims.every(dim => {
+      const col = (this.fields[dim] || {}).label || dim;
+      return this._store.dimensions.includes(col);
+    });
+    if (!hasAllFilterDims) return null;
+
+    const rows = this._store.rows();
+    return filterDims.length > 0
+      ? this._filterRows(rows, activeFilters)
+      : rows;
+  }
+
+  async getRowsForDims(requiredDims, activeFilters = {}) {
+    const cached = this.getBestRows(requiredDims, activeFilters);
+    if (cached) return { rows: cached, fromCache: true };
+
+    const rows = await this._fetchGroupBy(requiredDims, activeFilters);
+    return { rows, fromCache: false };
+  }
+
+  /**
+   * Filters rows from cache by active filters (no server request).
+   */
+  _filterRows(rows, activeFilters) {
+    const predicates = [];
+    for (const [dim, filter] of Object.entries(activeFilters)) {
+      const col = (this.fields[dim] || {}).label || dim;
+      if (filter.values && filter.values.length > 0) {
+        const valSet = new Set(filter.values);
+        predicates.push(row => valSet.has(String(row[col] ?? '')));
+      }
+      if (filter.searchText) {
+        const text = filter.searchText.toLowerCase();
+        predicates.push(filter.searchType === 'starts_with'
+          ? row => String(row[col] ?? '').toLowerCase().startsWith(text)
+          : row => String(row[col] ?? '').toLowerCase().includes(text)
+        );
+      }
+    }
+    if (!predicates.length) return rows;
+
+    // Materialise into array — stable and predictable
+    const filtered = [];
+    for (const row of rows) {
+      if (predicates.every(p => p(row))) filtered.push(row);
+    }
+    return filtered;
+  }
+
+  // ── Drillthrough ───────────────────────────────────────────────────────────
+
+  async countDistinct(logicalField) {
+    const col = (this.fields[logicalField] || {}).label || logicalField;
+    const sql = `SELECT COUNT(DISTINCT ${col}) AS cnt FROM (${this.query}) _t`;
+    const rows = await this._execute(sql);
+    return Number(rows[0]?.cnt || 0);
+  }
+
+  async getDistinctValues(logicalField) {
+    const def = this.fields[logicalField] || {};
+    const col = def.label || logicalField;
+    const sortCol = def.sortKey || col;
+    const sql = sortCol !== col
+      ? `SELECT DISTINCT ${col}, ${sortCol} FROM (${this.query}) _t ORDER BY ${sortCol}`
+      : `SELECT DISTINCT ${col} FROM (${this.query}) _t ORDER BY ${sortCol}`;
+    const rows = await this._execute(sql);
+    return rows.map(r => String(r[col] ?? ''));
+  }
+
+  async drillthrough({ filters = {} }) {
+    const where = this._buildWhere(filters);
+    const sql = this.drillthroughQuery
+      ? this.drillthroughQuery.replace('{filters}', where ? where.replace('WHERE ', '') : '1=1')
+      : `SELECT * FROM (${this.query}) _t ${where} LIMIT 200`;
+    return this._execute(sql);
+  }
+
+  // ── SQL helpers ────────────────────────────────────────────────────────────
+
+  _fetchGroupBy(logicalFields, activeFilters = {}) {
+    const select = [];
+    const groupBy = [];
+    const orderBy = [];
+
+    for (const field of logicalFields) {
+      const def = this.fields[field] || {};
+      if (def.sortKey) {
+        select.push(def.sortKey, def.label);
+        groupBy.push(def.sortKey, def.label);
+        orderBy.push(def.sortKey);
+      } else {
+        const col = def.label || field;
+        select.push(col);
+        groupBy.push(col);
+        orderBy.push(col);
+      }
+    }
+
+    const aggExprs = this.measures.flatMap(m =>
+      this.funcs.map(fn => `${fn.toUpperCase()}(${m}) AS ${m}_${fn}`)
+    ).join(', ');
+
+    const where = this._buildFiltersWhere(activeFilters);
+
+    const sql = `
+      SELECT ${[...select, aggExprs].join(', ')}
+      FROM   (${this.query}) _t
+      ${where}
+      GROUP  BY ${groupBy.join(', ')}
+      ORDER  BY ${orderBy.join(', ')}
+    `;
+
+    return this._execute(sql);
+  }
+
+  /** Builds a WHERE clause for active filters (for SQL queries). */
+  _buildFiltersWhere(activeFilters = {}) {
+    const conditions = [];
+    for (const [dim, filter] of Object.entries(activeFilters)) {
+      const col = (this.fields[dim] || {}).label || dim;
+
+      if (filter.values && filter.values.length > 0) {
+        const vals = filter.values
+          .map(v => `'${String(v).replace(/'/g, "''")}'`)
+          .join(', ');
+        conditions.push(`${col} IN (${vals})`);
+      }
+
+      if (filter.searchText) {
+        const esc = filter.searchText.replace(/'/g, "''");
+        conditions.push(filter.searchType === 'starts_with'
+          ? `${col} ILIKE '${esc}%'`
+          : `${col} ILIKE '%${esc}%'`
+        );
+      }
+    }
+    return conditions.length ? 'WHERE ' + conditions.join(' AND ') : '';
+  }
+
+  /** Expands logical field names into real DB column names. */
+  _expandFields(logicalFields) {
+    const cols = [];
+    for (const field of logicalFields) {
+      const def = this.fields[field] || {};
+      if (def.sortKey) cols.push(def.sortKey);
+      cols.push(def.label || field);
+    }
+    return cols;
+  }
+
+  _makeStore(logicalFields, rows) {
+    const dims = this._expandFields(logicalFields);
+    const store = new ColumnStore({
+      dimensions: dims,
+      measures: this.measures,
+      funcs: this.funcs,
+      capacity: this.maxCachedRows,
+    });
+    store.append(rows);
+    return store;
+  }
+
+  _buildWhere(filters) {
+    const keys = Object.keys(filters);
+    if (!keys.length) return '';
+    return 'WHERE ' + keys.map(k => {
+      const def = this.fields[k] || {};
+      const col = def.label || k;
+      return `${col} = '${String(filters[k]).replace(/'/g, "''")}'`;
+    }).join(' AND ');
+  }
+
+  // ── HTTP ───────────────────────────────────────────────────────────────────
+
+  async _execute(query) {
+    const res = await fetch(this.url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ query }),
+    });
+
+    if (!res.ok) {
+      const err = await res.json().catch(() => ({}));
+      throw new Error(`Server error ${res.status}: ${err.error || ''}`);
+    }
+
+    const rows = await res.json();
+    return rows.map(row => {
+      const out = {};
+      for (const k of Object.keys(row)) out[k.toLowerCase()] = row[k];
+      return out;
+    });
+  }
+
+  async load() {
+    throw new Error('Use prefetch() / getRowsForDims() / drillthrough()');
+  }
+}

package/server/.env

@@ -0,0 +1,5 @@
+DB_HOST=localhost
+DB_PORT=5432
+DB_NAME=postgres
+DB_USER=postgres
+DB_PASSWORD=admin

package/server/README.md

@@ -0,0 +1,88 @@
+# PivotGrid Server
+
+A lightweight Python proxy between PivotGrid and your database.
+
+## Requirements
+
+- Python 3.8+
+- psycopg2-binary (for PostgreSQL)
+
+## Installation
+
+```bash
+pip install psycopg2-binary
+```
+
+## Start
+
+```bash
+python server.py
+```
+
+Server starts on `http://localhost:8000` by default.
+
+## Configuration
+
+Settings are stored in a `.env` file in the `server/` directory:
+
+```env
+PORT=8000
+
+DB_CONNECTOR=postgresql
+DB_HOST=localhost
+DB_PORT=5432
+DB_NAME=postgres
+DB_USER=postgres
+DB_PASSWORD=secret
+```
+
+All settings can also be configured via the Config Editor UI.
+
+## Endpoints
+
+| Method | Path | Description |
+|--------|------|-------------|
+| `POST` | `/query` | Execute a SELECT query, returns JSON array |
+| `GET` | `/configs` | List all config names |
+| `GET` | `/configs/{name}` | Get a config by name |
+| `POST` | `/configs/{name}` | Save a config by name |
+| `GET` | `/server-config` | Get DB settings (password excluded) |
+| `POST` | `/server-config` | Save DB settings to `.env` |
+| `POST` | `/test-connection` | Test DB connection with given credentials |
+
+## Configs
+
+Configs are stored as JSON files in `server/configs/`:
+
+```
+server/
+└── configs/
+    ├── sales.json
+    ├── orders.json
+    └── ...
+```
+
+## Adding a Custom DB Connector
+
+Create a file in `server/connectors/`:
+
+```python
+# server/connectors/mydb.py
+NAME = "My Database"
+
+def execute_query(query):
+    # your implementation
+    return [{"col": "val"}, ...]
+
+def test_connection(host, port, dbname, user, password):
+    # optional — enables "Test connection" button in Config Editor
+    pass
+```
+
+The server picks up the new connector on next startup.
+
+## Notes
+
+- Only `SELECT` queries are allowed via `/query`
+- Password is never returned by `/server-config`
+- CORS is enabled for all origins (`*`) — restrict in production if needed

package/server/configs/main_config.json

@@ -0,0 +1,112 @@
+{
+  "query": "\n    SELECT * FROM sales_data\n  ",
+  "dimensions": [
+    "region",
+    "category",
+    "product",
+    "manager",
+    "channel",
+    "sale_year",
+    "sale_quarter",
+    "sale_month_name",
+    "sale_week",
+    "sale_day_num",
+    "sale_weekday_name",
+    "sale_hour",
+    "sale_minute"
+  ],
+  "measures": [
+    "revenue",
+    "units"
+  ],
+  "funcs": [
+    "sum",
+    "avg",
+    "count",
+    "min",
+    "max",
+    "stddev",
+    "variance"
+  ],
+  "fields": {
+    "region": {
+      "label": "region",
+      "title": "Region"
+    },
+    "category": {
+      "label": "category",
+      "title": "Category"
+    },
+    "product": {
+      "label": "product",
+      "title": "Product"
+    },
+    "manager": {
+      "label": "manager",
+      "title": "Manager"
+    },
+    "channel": {
+      "label": "channel",
+      "title": "Channel"
+    },
+    "revenue": {
+      "label": "revenue",
+      "title": "Revenue"
+    },
+    "units": {
+      "label": "units",
+      "title": "Units"
+    },
+    "sale_year": {
+      "label": "sale_year",
+      "title": "Year"
+    },
+    "sale_quarter": {
+      "label": "sale_quarter",
+      "title": "Quarter"
+    },
+    "sale_month_name": {
+      "label": "sale_month_name",
+      "title": "Month",
+      "sortKey": "sale_month_num"
+    },
+    "sale_week": {
+      "label": "sale_week",
+      "title": "Week"
+    },
+    "sale_day_num": {
+      "label": "sale_day_num",
+      "title": "Day"
+    },
+    "sale_weekday_name": {
+      "label": "sale_weekday_name",
+      "title": "Weekday",
+      "sortKey": "sale_weekday_num"
+    },
+    "sale_hour": {
+      "label": "sale_hour",
+      "title": "Hour"
+    },
+    "sale_minute": {
+      "label": "sale_minute",
+      "title": "Minute"
+    }
+  },
+  "cachedDimensions": [
+    "sale_year",
+    "sale_month_name"
+  ],
+  "rows": [
+    "region",
+    "channel"
+  ],
+  "columns": [
+    "sale_year",
+    "sale_month_name"
+  ],
+  "measure": "revenue",
+  "func": "sum",
+  "maxCachedRows": 500000,
+  "filterCheckboxLimit": 30,
+  "drillthroughQuery": "\n    SELECT ID,REGION FROM sales_data\n    WHERE {filters}\n    LIMIT 200 OFFSET 0\n  "
+}

package/server/connectors/postgresql.py

@@ -0,0 +1,34 @@
+"""
+PostgreSQL connector.
+
+Installation:
+    pip install psycopg2-binary
+"""
+
+import os
+import psycopg2
+import psycopg2.extras
+
+NAME = "PostgreSQL"
+
+def execute_query(query):
+    conn = psycopg2.connect(
+        host=os.getenv("DB_HOST",     "localhost"),
+        port=int(os.getenv("DB_PORT", "5432")),
+        dbname=os.getenv("DB_NAME",   "postgres"),
+        user=os.getenv("DB_USER",     "postgres"),
+        password=os.getenv("DB_PASSWORD", ""),
+    )
+    cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+    cur.execute(query)
+    rows = [dict(r) for r in cur.fetchall()]
+    cur.close()
+    conn.close()
+    return rows
+
+def test_connection(host, port, dbname, user, password):
+    conn = psycopg2.connect(
+        host=host, port=int(port), dbname=dbname,
+        user=user, password=password,
+    )
+    conn.close()