spice-js 2.6.65 → 2.6.67

package/build/models/SpiceModel.js

@@ -916,25 +916,76 @@ class SpiceModel {
     JSON.stringify(this);
   }
 
-  prepColumns(columns) {
-    if (columns && columns !== "") {
-      var columnList = columns.split(",");
-      return _.join(_.compact(columnList.map(column => {
-        if (column === "meta().id") return undefined;
+  buildArrayProjection(alias, field) {
+    var safeAlias = String(alias).replace(/`/g, "");
+    var safeField = String(field).replace(/`/g, "");
+    return "ARRAY rc." + safeField + " FOR rc IN IFMISSINGORNULL(" + safeAlias + ", []) END AS `" + safeAlias + "_" + safeField + "`";
+  }
+
+  prepColumns(columns, protectedAliases, arrayAliases) {
+    if (protectedAliases === void 0) {
+      protectedAliases = [];
+    }
+
+    if (arrayAliases === void 0) {
+      arrayAliases = [];
+    }
+
+    if (!columns || columns === "") return columns;
+    var protectedSet = new Set(protectedAliases);
+    var arraySet = new Set(arrayAliases);
+    var tokens = columns.split(",");
+    var out = tokens.map(raw => {
+      var col = (raw || "").trim();
+      if (col === "" || col === "meta().id") return undefined;
+      if (/^\s*ARRAY\s+/i.test(col) || /\w+\s*\(/.test(col)) return col;
+      var m = col.match(/^\s*`?(\w+)`?\.`?(\w+)`?(?:\s+AS\s+`?([\w]+)`?)?\s*$/i);
+
+      if (m) {
+        var alias = m[1];
+        var field = m[2];
+        var explicitAs = m[3];
+
+        if (arraySet.has(alias)) {
+          var proj = this.buildArrayProjection(alias, field);
+
+          if (explicitAs && explicitAs !== alias + "_" + field) {
+            proj = proj.replace(/AS\s+`[^`]+`$/i, "AS `" + explicitAs + "`");
+          }
 
-        if (!column.startsWith("`") && !column.endsWith("`")) {
-          column = "`" + column.trim() + "`";
+          return proj;
         }
 
-        if (column && !column.includes(".") && !column.startsWith(this.type)) {
-          column = "`" + this.type + "`." + column.trim();
+        if (protectedSet.has(alias)) {
+          var aliased = "`" + alias + "`." + (field.startsWith("`") ? field : "`" + field + "`");
+          return explicitAs ? aliased + " AS `" + explicitAs + "`" : aliased;
         }
 
-        return column;
-      })), ",");
-    }
+        var bare = col.replace(/`/g, "");
+        return "`" + this.type + "`." + bare;
+      }
+
+      var looksProtected = [...protectedSet].some(a => col === a || col === "`" + a + "`" || col.startsWith(a + ".") || col.startsWith("`" + a + "`."));
 
-    return columns;
+      if (!looksProtected) {
+        if (!col.startsWith("`") && !col.endsWith("`") && !col.includes("(")) {
+          col = "`" + col + "`";
+        }
+
+        if (col && !col.includes(".") && !col.startsWith(this.type)) {
+          col = "`" + this.type + "`." + col.replace(/`/g, "");
+        }
+
+        return col;
+      }
+
+      if (!col.includes(".") && !col.startsWith("`")) {
+        return "`" + col + "`";
+      }
+
+      return col;
+    });
+    return _.join(_.compact(out), ",");
   }
 
   filterResultsByColumns(data, columns) {
@@ -989,12 +1040,20 @@ class SpiceModel {
     return [...new Set(returnVal)];
   }
 
-  createJoinSection(nestings) {
-    return nestings.map(nesting => {
-      if (nesting.type === _2.DataType.ARRAY || nesting.type === Array || nesting.type === "array") {
-        return "LEFT NEST `" + (0, _fix.fixCollection)(nesting.reference) + "` AS `" + nesting.alias + "` ON KEYS `" + this.type + "`.`" + nesting.alias + "`";
+  createJoinSection(mappedNestings) {
+    if (!mappedNestings || mappedNestings.length === 0) return "";
+    return mappedNestings.map((_ref2) => {
+      var {
+        alias,
+        reference,
+        is_array
+      } = _ref2;
+      var keyspace = (0, _fix.fixCollection)(reference);
+
+      if (is_array === true) {
+        return "LEFT NEST `" + keyspace + "` AS `" + alias + "` ON KEYS `" + this.type + "`.`" + alias + "`";
       } else {
-        return "LEFT JOIN `" + (0, _fix.fixCollection)(nesting.reference) + "` AS `" + nesting.alias + "` ON KEYS `" + this.type + "`.`" + nesting.alias + "`";
+        return "LEFT JOIN `" + keyspace + "` AS `" + alias + "` ON KEYS `" + this.type + "`.`" + alias + "`";
       }
     }).join(" ");
   }
@@ -1009,67 +1068,63 @@ class SpiceModel {
   } */
 
 
-  fixColumnName(columns) {
-    // Guard clause: if columns is not provided or not a string, return it as-is.
-    if (!columns || typeof columns !== "string") {
-      return columns;
-    } // Split the columns string on commas and trim each column expression.
+  fixColumnName(columns, protectedAliases) {
+    if (protectedAliases === void 0) {
+      protectedAliases = [];
+    }
+
+    if (!columns || typeof columns !== "string") return columns;
+    var protectedSet = new Set(protectedAliases);
+    var tokens = columns.split(",").map(s => s.trim());
+    var aliasTracker = {};
+    var out = tokens.map(col => {
+      if (!col) return undefined; // Do not rewrite ARRAY projections
 
+      if (/^\s*ARRAY\s+/i.test(col)) return col; // If token is literally this.type.alias → compress to `alias`
+
+      for (var a of protectedSet) {
+        var re = new RegExp("^`?" + _.escapeRegExp(this.type) + "`?\\.`?" + _.escapeRegExp(a) + "`?$");
+
+        if (re.test(col)) {
+          return "`" + a + "`";
+        }
+      } // Extract explicit AS
 
-    var columnList = columns.split(",").map(col => col.trim()); // Object to keep track of alias usage to avoid duplicates.
 
-    var aliasTracker = {};
-    var fixedColumns = columnList.map(col => {
-      // Check if an explicit alias is already provided.
       var aliasRegex = /\s+AS\s+`?([\w]+)`?$/i;
       var aliasMatch = col.match(aliasRegex);
-      var explicitAlias = aliasMatch ? aliasMatch[1] : null; // Remove the alias clause for easier processing.
+      var explicitAlias = aliasMatch ? aliasMatch[1] : null;
+      var colWithoutAlias = explicitAlias ? col.replace(aliasRegex, "").trim() : col; // `table`.`col` or table.col
 
-      var colWithoutAlias = explicitAlias ? col.replace(aliasRegex, "").trim() : col; // Use regex to extract the table and column names.
-      // This regex matches optional backticks around each identifier.
-
-      var tableName = this.type;
-      var columnName = "";
       var columnRegex = /^`?([\w]+)`?\.`?([\w]+)`?$/;
       var match = colWithoutAlias.match(columnRegex);
+      var tableName = this.type;
+      var columnName = colWithoutAlias.replace(/`/g, "");
 
       if (match) {
         tableName = match[1];
         columnName = match[2];
-      } else {
-        // If no table prefix is found, use the column name only.
-        columnName = colWithoutAlias.replace(/`/g, "");
-      } // Generate the alias.
-      // If the column comes from a table different from this.type, then the alias
-      // becomes tableName_columnName. Otherwise, just use the column name.
-
+      } // For joined table columns, add default alias <table_col> if none
 
-      var newAlias = columnName;
 
       if (tableName && tableName !== this.type) {
-        newAlias = tableName + "_" + columnName; // If an explicit alias was provided, favor that.
-
-        if (explicitAlias) {
-          newAlias = explicitAlias;
-        } // Avoid duplicates by appending a count if needed.
-
-
-        if (aliasTracker.hasOwnProperty(newAlias)) {
-          aliasTracker[newAlias]++;
-          newAlias = newAlias + "_" + aliasTracker[newAlias];
-        } else {
-          aliasTracker[newAlias] = 0;
-        } // If there's no explicit alias, add an alias clause.
-
+        var newAlias = explicitAlias || tableName + "_" + columnName;
 
         if (!explicitAlias) {
+          if (aliasTracker.hasOwnProperty(newAlias)) {
+            aliasTracker[newAlias]++;
+            newAlias = newAlias + "_" + aliasTracker[newAlias];
+          } else {
+            aliasTracker[newAlias] = 0;
+          }
+
           return colWithoutAlias + " AS `" + newAlias + "`";
         }
       }
 
       return col;
     });
-    return fixedColumns.join(", ");
+    return _.join(_.compact(out), ", ");
   }
 
   list(args) {
@@ -1083,38 +1138,55 @@ class SpiceModel {
       try {
         var _args6, _args7, _args8;
 
-        args.columns = _this12.prepColumns(args.columns);
-        if (args.mapping_dept) _this12[_mapping_dept] = args.mapping_dept;
-        var nestings = [..._this12.extractNestings((_args6 = args) == null ? void 0 : _args6.query, _this12.type), ..._this12.extractNestings((_args7 = args) == null ? void 0 : _args7.columns, _this12.type), ..._this12.extractNestings((_args8 = args) == null ? void 0 : _args8.sort, _this12.type)];
-        args.columns = _this12.fixColumnName(args.columns);
+        if (args.mapping_dept) _this12[_mapping_dept] = args.mapping_dept; // Find alias tokens from query/columns/sort
 
-        var mappedNestings = _.compact(_.uniq(nestings).map(nesting => {
-          var _prop$map;
+        var nestings = [..._this12.extractNestings(((_args6 = args) == null ? void 0 : _args6.query) || "", _this12.type), ..._this12.extractNestings(((_args7 = args) == null ? void 0 : _args7.columns) || "", _this12.type), ..._this12.extractNestings(((_args8 = args) == null ? void 0 : _args8.sort) || "", _this12.type)]; // Decide which aliases we can join: only when map.type===MODEL AND reference is a STRING keyspace.
 
-          var prop = _this12.props[nesting];
+        var mappedNestings = _.compact(_.uniq(nestings).map(alias => {
+          var prop = _this12.props[alias];
+          if (!(prop == null ? void 0 : prop.map) || prop.map.type !== _2.MapType.MODEL) return null;
+          var ref = prop.map.reference;
 
-          if ((prop == null ? void 0 : (_prop$map = prop.map) == null ? void 0 : _prop$map.type) === _2.MapType.MODEL) {
-            return {
-              alias: nesting,
-              reference: prop.map.reference.toLowerCase(),
-              type: prop.type,
-              value_field: prop.map.value_field
-            };
+          if (typeof ref !== "string") {
+            // reference is a class/function/array-of-classes → no SQL join; serializer will handle it
+            return null;
           }
+
+          var is_array = prop.type === "array" || prop.type === Array || prop.type === _2.DataType.ARRAY;
+          return {
+            alias,
+            reference: ref.toLowerCase(),
+            // keyspace to join
+            is_array,
+            type: prop.type,
+            value_field: prop.map.value_field,
+            destination: prop.map.destination || alias
+          };
         }));
 
-        args._join = _this12.createJoinSection(mappedNestings);
+        var protectedAliases = mappedNestings.map(m => m.alias);
+        var arrayAliases = mappedNestings.filter(m => m.is_array).map(m => m.alias); // Columns: first prepare (prefix base table, rewrite array alias.field → ARRAY proj),
+        // then normalize names and add default aliases
+
+        args.columns = _this12.prepColumns(args.columns, protectedAliases, arrayAliases);
+        args.columns = _this12.fixColumnName(args.columns, protectedAliases); // Build JOIN/NEST from the mapped keyspaces
+
+        args._join = _this12.createJoinSection(mappedNestings); // WHERE
+
         var query = "";
+        var deletedCondition = "(`" + _this12.type + "`.deleted = false OR `" + _this12.type + "`.deleted IS MISSING)";
 
         if (args.is_full_text === "true" || args.is_custom_query === "true") {
-          query = args.query;
+          query = args.query || "";
+        } else if (args.filters) {
+          query = _this12.makeQueryFromFilter(args.filters);
+        } else if (args.query) {
+          query = args.query + " AND " + deletedCondition;
         } else {
-          query = args.filters ? _this12.makeQueryFromFilter(args.filters) : args.query ? args.query + " AND (`" + _this12.type + "`.deleted = false OR `" + _this12.type + "`.deleted IS MISSING)" : "(`" + _this12.type + "`.deleted = false OR `" + _this12.type + "`.deleted IS MISSING)";
+          query = deletedCondition;
         }
 
-        if ((0, _Security.hasSQLInjection)(query)) {
-          return [];
-        }
+        if ((0, _Security.hasSQLInjection)(query)) return []; // LIMIT/OFFSET/SORT
 
         args.limit = Number(args.limit) || undefined;
         args.offset = Number(args.offset) || 0;
@@ -1128,15 +1200,12 @@ class SpiceModel {
         var results;
 
         if (_this12.shouldUseCache(_this12.type)) {
-          var cachedResults = yield _this12.getCacheProviderObject(_this12.type).get(cacheKey);
-          results = cachedResults == null ? void 0 : cachedResults.value; // Check if there is no cached value
+          var cached = yield _this12.getCacheProviderObject(_this12.type).get(cacheKey);
+          results = cached == null ? void 0 : cached.value;
+          var isEmpty = (cached == null ? void 0 : cached.value) === undefined;
+          var refresh = yield _this12.shouldForceRefresh(cached);
 
-          var isCacheEmpty = (cachedResults == null ? void 0 : cachedResults.value) === undefined; // Check if the cached value should be refreshed (e.g., due to the monitor's timestamp)
-
-          var shouldRefresh = yield _this12.shouldForceRefresh(cachedResults); // Always force a refresh for "workflow" type
-
-          if (isCacheEmpty || shouldRefresh) {
-            // Force a database read and cache update
+          if (isEmpty || refresh) {
             results = yield _this12.fetchResults(args, query);
 
             _this12.getCacheProviderObject(_this12.type).set(cacheKey, {
@@ -1146,7 +1215,8 @@ class SpiceModel {
           }
         } else {
           results = yield _this12.fetchResults(args, query);
-        }
+        } // Serializer still handles class-based refs and value_field
+
 
         if (args.skip_read_serialize !== true && args.skip_serialize !== true) {
           results.data = yield _this12.do_serialize(results.data, "read", {}, args, (yield _this12.propsToBeRemoved(results.data)));
@@ -1180,12 +1250,12 @@ class SpiceModel {
     })();
   }
 
-  addHook(_ref2) {
+  addHook(_ref3) {
     var {
       operation,
       when,
       execute
-    } = _ref2;
+    } = _ref3;
 
     this[_hooks][operation][when].push(execute);
   }
@@ -1362,11 +1432,11 @@ class SpiceModel {
     })();
   }
 
-  addModifier(_ref3) {
+  addModifier(_ref4) {
     var {
       when,
       execute
-    } = _ref3;
+    } = _ref4;
 
     if (this[_serializers][when]) {
       this[_serializers][when]["modifiers"].push(execute);

package/build/utility/RestHelper.js

@@ -11,6 +11,8 @@ function _getRequireWildcardCache() { if (typeof WeakMap !== "function") return
 
 function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
 
+function _extends() { _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; return _extends.apply(this, arguments); }
+
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
 
 function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
@@ -66,55 +68,154 @@ class RestHelper {
 
   static send_download(ctx, next) {
     return _asyncToGenerator(function* () {
-      function makeDirectory(dir) {
-        if (!fs.existsSync(dir)) {
-          fs.mkdirSync(dir);
+      var makeDirectory = dir => {
+        if (!fs.existsSync(dir)) fs.mkdirSync(dir, {
+          recursive: true
+        });
+      };
+
+      var deepTrimStrings = val => {
+        if (Array.isArray(val)) return val.map(deepTrimStrings);
+
+        if (val && typeof val === "object") {
+          var out = {};
+
+          for (var [k, v] of Object.entries(val)) {
+            out[k] = deepTrimStrings(v);
+          }
+
+          return out;
         }
-      }
 
-      try {
-        var download_type = ctx.request.query.format || "csv";
-        var include_id = ctx.request.query.include_id;
-        var content;
+        return typeof val === "string" ? val.trim() : val;
+      };
 
-        if (download_type == "csv") {
-          var {
-            flatten
-          } = yield Promise.resolve().then(() => _interopRequireWildcard(require("flat")));
+      var stripTopLevel = (obj, _ref) => {
+        var {
+          removeId
+        } = _ref;
+        if (Array.isArray(obj)) return obj.map(i => stripTopLevel(i, {
+          removeId
+        }));
 
-          var items = _lodash.default.map(ctx.data, item => {
-            if (!include_id || include_id == "false") {
-              delete item.id;
-            }
+        if (obj && typeof obj === "object") {
+          var copy = _extends({}, obj);
 
-            delete item._permissions_;
-            return flatten(item);
-          });
+          if (removeId) delete copy.id;
+          delete copy._permissions_;
+          return copy;
+        }
 
-          var fields = _lodash.default.union(_lodash.default.keys(_lodash.default.first(items)), _lodash.default.keys(_lodash.default.last(items)), _lodash.default.keys(_lodash.default.nth(items.length / 2)));
+        return obj;
+      };
 
-          var opts = {
-            fields
-          };
-          content = parse(items, opts);
-          ctx.set("content-type", "text/csv");
-        } else {
-          content = JSON.stringify(ctx.data);
-          ctx.set("content-type", "application/json");
+      var normalizeEmptyArraysForCsv = val => {
+        if (Array.isArray(val)) {
+          if (val.length === 0) return undefined;
+          return val.map(v => normalizeEmptyArraysForCsv(v));
         }
 
-        makeDirectory("./storage/exports");
-        makeDirectory("./storage/exports/" + download_type + "/");
-        var file = path.resolve("./storage/exports/" + download_type + "/" + RestHelper.makeid(9) + "." + download_type);
-        yield fs.writeFile(file, content, function (err) {
-          if (err) throw err;
+        if (val && typeof val === "object") {
+          var out = {};
+
+          for (var [k, v] of Object.entries(val)) {
+            var nv = normalizeEmptyArraysForCsv(v);
+            if (nv !== undefined) out[k] = nv;
+          }
+
+          return out;
+        }
+
+        return val;
+      };
+
+      var safeJSONStringify = function safeJSONStringify(value, space) {
+        if (space === void 0) {
+          space = 2;
+        }
+
+        var seen = new WeakSet();
+
+        var replacer = (_k, v) => {
+          if (typeof v === "bigint") return v.toString();
+          if (v instanceof Date) return v.toISOString();
+
+          if (v && typeof v === "object") {
+            if (seen.has(v)) return "[Circular]";
+            seen.add(v);
+          }
+
+          return v;
+        };
+
+        return JSON.stringify(value, replacer, space);
+      };
+
+      try {
+        var download_type = (ctx.request.query.format || "csv").toLowerCase();
+        var include_id = ctx.request.query.include_id;
+
+        var original = _lodash.default.cloneDeep(ctx.data);
+
+        var trimmed = deepTrimStrings(original);
+        var cleaned = stripTopLevel(trimmed, {
+          removeId: !include_id || include_id === "false"
         });
-        ctx.set("content-disposition", "attachment");
-        ctx.response.attachment(file);
+        var filename, filePath;
+
+        if (download_type === "csv") {
+          var _ret = yield* function* () {
+            var {
+              flatten
+            } = yield Promise.resolve().then(() => _interopRequireWildcard(require("flat")));
+            var rows = Array.isArray(cleaned) ? cleaned : [cleaned];
+            var csvReady = rows.map(normalizeEmptyArraysForCsv);
+            var flatRows = csvReady.map(row => flatten(row, {
+              safe: false,
+              delimiter: "."
+            }));
+            var fieldSet = new Set();
+
+            for (var r of flatRows) {
+              Object.keys(r).forEach(k => fieldSet.add(k));
+            }
+
+            var fields = Array.from(fieldSet).sort((a, b) => a.localeCompare(b, undefined, {
+              numeric: true,
+              sensitivity: "base"
+            }));
+            var csv = parse(flatRows, {
+              fields,
+              defaultValue: "",
+              excelStrings: true
+            });
+            makeDirectory("./storage/exports/csv");
+            filename = RestHelper.makeid(9) + ".csv";
+            filePath = path.resolve("./storage/exports/csv/" + filename);
+            yield fs.promises.writeFile(filePath, csv, "utf8");
+            ctx.set("Content-Disposition", "attachment; filename=\"" + filename + "\"");
+            ctx.type = "text/csv; charset=utf-8";
+            ctx.status = 200;
+            ctx.body = fs.createReadStream(filePath);
+            return {
+              v: void 0
+            };
+          }();
+
+          if (typeof _ret === "object") return _ret.v;
+        }
+
+        var jsonText = safeJSONStringify(cleaned, 2);
+        makeDirectory("./storage/exports/json");
+        filename = RestHelper.makeid(9) + ".json";
+        filePath = path.resolve("./storage/exports/json/" + filename);
+        yield fs.promises.writeFile(filePath, jsonText, "utf8");
+        ctx.set("Content-Disposition", "attachment; filename=\"" + filename + "\"");
+        ctx.type = "application/json; charset=utf-8";
         ctx.status = 200;
-        ctx.body = fs.createReadStream(file);
+        ctx.body = fs.createReadStream(filePath);
       } catch (e) {
-        console.log(e.stack);
+        console.error(e.stack);
         ctx.status = 400;
         ctx.body = RestHelper.prepare_response(RestHelper.FAILURE, e);
       }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spice-js",
-  "version": "2.6.65",
+  "version": "2.6.67",
   "description": "spice",
   "main": "build/index.js",
   "repository": {

package/src/utility/RestHelper.js

@@ -41,57 +41,112 @@ export default class RestHelper {
   }
 
   static async send_download(ctx, next) {
-    function makeDirectory(dir) {
-      if (!fs.existsSync(dir)) {
-        fs.mkdirSync(dir);
+    const makeDirectory = (dir) => {
+      if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    };
+  
+    const deepTrimStrings = (val) => {
+      if (Array.isArray(val)) return val.map(deepTrimStrings);
+      if (val && typeof val === "object") {
+        const out = {};
+        for (const [k, v] of Object.entries(val)) out[k] = deepTrimStrings(v);
+        return out;
       }
-    }
+      return typeof val === "string" ? val.trim() : val;
+    };
+  
+    const stripTopLevel = (obj, { removeId }) => {
+      if (Array.isArray(obj)) return obj.map((i) => stripTopLevel(i, { removeId }));
+      if (obj && typeof obj === "object") {
+        const copy = { ...obj };
+        if (removeId) delete copy.id;
+        delete copy._permissions_;
+        return copy;
+      }
+      return obj;
+    };
+  
+    const normalizeEmptyArraysForCsv = (val) => {
+      if (Array.isArray(val)) {
+        if (val.length === 0) return undefined;
+        return val.map((v) => normalizeEmptyArraysForCsv(v));
+      }
+      if (val && typeof val === "object") {
+        const out = {};
+        for (const [k, v] of Object.entries(val)) {
+          const nv = normalizeEmptyArraysForCsv(v);
+          if (nv !== undefined) out[k] = nv;
+        }
+        return out;
+      }
+      return val;
+    };
+  
+    const safeJSONStringify = (value, space = 2) => {
+      const seen = new WeakSet();
+      const replacer = (_k, v) => {
+        if (typeof v === "bigint") return v.toString();
+        if (v instanceof Date) return v.toISOString();
+        if (v && typeof v === "object") {
+          if (seen.has(v)) return "[Circular]";
+          seen.add(v);
+        }
+        return v;
+      };
+      return JSON.stringify(value, replacer, space);
+    };
+  
     try {
-      let download_type = ctx.request.query.format || "csv";
-      let include_id = ctx.request.query.include_id;
-      let content;
-      if (download_type == "csv") {
+      const download_type = (ctx.request.query.format || "csv").toLowerCase();
+      const include_id = ctx.request.query.include_id;
+  
+      const original = _.cloneDeep(ctx.data);
+      const trimmed  = deepTrimStrings(original);
+      const cleaned  = stripTopLevel(trimmed, { removeId: !include_id || include_id === "false" });
+  
+      let filename, filePath;
+  
+      if (download_type === "csv") {
         const { flatten } = await import("flat");
-
-        let items = _.map(ctx.data, (item) => {
-          if (!include_id || include_id == "false") {
-            delete item.id;
-          }
-          delete item._permissions_;
-          return flatten(item);
-        });
-        let fields = _.union(
-          _.keys(_.first(items)),
-          _.keys(_.last(items)),
-          _.keys(_.nth(items.length / 2))
+        const rows = Array.isArray(cleaned) ? cleaned : [cleaned];
+  
+        const csvReady = rows.map(normalizeEmptyArraysForCsv);
+        const flatRows = csvReady.map((row) => flatten(row, { safe: false, delimiter: "." }));
+  
+        const fieldSet = new Set();
+        for (const r of flatRows) Object.keys(r).forEach((k) => fieldSet.add(k));
+        const fields = Array.from(fieldSet).sort((a, b) =>
+          a.localeCompare(b, undefined, { numeric: true, sensitivity: "base" })
         );
-        const opts = {
-          fields,
-        };
-        content = parse(items, opts);
-        ctx.set("content-type", "text/csv");
-      } else {
-        content = JSON.stringify(ctx.data);
-        ctx.set("content-type", "application/json");
+  
+        const csv = parse(flatRows, { fields, defaultValue: "", excelStrings: true });
+  
+        makeDirectory(`./storage/exports/csv`);
+        filename = `${RestHelper.makeid(9)}.csv`;
+        filePath = path.resolve(`./storage/exports/csv/${filename}`);
+        await fs.promises.writeFile(filePath, csv, "utf8");
+    
+ 
+        ctx.set("Content-Disposition", `attachment; filename="${filename}"`);
+        ctx.type = "text/csv; charset=utf-8";
+        ctx.status = 200;
+        ctx.body = fs.createReadStream(filePath);
+        return;
       }
 
-      makeDirectory(`./storage/exports`);
-      makeDirectory(`./storage/exports/${download_type}/`);
-      let file = path.resolve(
-        `./storage/exports/${download_type}/${RestHelper.makeid(
-          9
-        )}.${download_type}`
-      );
-      await fs.writeFile(file, content, function (err) {
-        if (err) throw err;
-      });
-
-      ctx.set("content-disposition", "attachment");
-      ctx.response.attachment(file);
+      const jsonText = safeJSONStringify(cleaned, 2);
+  
+      makeDirectory(`./storage/exports/json`);
+      filename = `${RestHelper.makeid(9)}.json`;
+      filePath = path.resolve(`./storage/exports/json/${filename}`);
+      await fs.promises.writeFile(filePath, jsonText, "utf8");
+      ctx.set("Content-Disposition", `attachment; filename="${filename}"`);
+      ctx.type = "application/json; charset=utf-8";
       ctx.status = 200;
-      ctx.body = fs.createReadStream(file);
+      ctx.body = fs.createReadStream(filePath);
+  
     } catch (e) {
-      console.log(e.stack);
+      console.error(e.stack);
       ctx.status = 400;
       ctx.body = RestHelper.prepare_response(RestHelper.FAILURE, e);
     }
@@ -203,3 +258,4 @@ export default class RestHelper {
     return false;
   }
 }
+