spice-js 2.7.32 → 2.7.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -32,7 +32,8 @@ var SDate = require("sonover-date"),
|
|
|
32
32
|
_mapping_concurrency = Symbol(),
|
|
33
33
|
_skip_mapped_serialization = Symbol(),
|
|
34
34
|
_mapping_dept_exempt = Symbol(),
|
|
35
|
-
_current_path = Symbol()
|
|
35
|
+
_current_path = Symbol(),
|
|
36
|
+
_failed_json_fields = Symbol();
|
|
36
37
|
var GLOBAL_MAPPING_CONCURRENCY = Math.max(1, Number(process.env.SPICE_MAPPING_GLOBAL_CONCURRENCY) || 32);
|
|
37
38
|
var activeGlobalMappings = 0;
|
|
38
39
|
var globalMappingWaiters = [];
|
|
@@ -71,7 +72,7 @@ function _acquireGlobalMappingSlot() {
|
|
|
71
72
|
}
|
|
72
73
|
function runWithConcurrency(_x, _x2, _x3) {
|
|
73
74
|
return _runWithConcurrency.apply(this, arguments);
|
|
74
|
-
}
|
|
75
|
+
}
|
|
75
76
|
function _runWithConcurrency() {
|
|
76
77
|
_runWithConcurrency = _asyncToGenerator(function* (items, concurrency, worker) {
|
|
77
78
|
var results = new Array(items.length);
|
|
@@ -97,6 +98,19 @@ function _runWithConcurrency() {
|
|
|
97
98
|
});
|
|
98
99
|
return _runWithConcurrency.apply(this, arguments);
|
|
99
100
|
}
|
|
101
|
+
var JSON_PARSE_FAILED = Symbol("json_parse_failed");
|
|
102
|
+
function safeParseJsonField(value, fieldName, modelType) {
|
|
103
|
+
if (value == null || value === "") return undefined;
|
|
104
|
+
try {
|
|
105
|
+
return JSON.parse(value);
|
|
106
|
+
} catch (e) {
|
|
107
|
+
console.warn("SpiceModel(" + modelType + "): failed to parse field \"" + fieldName + "\" as JSON: " + e.message);
|
|
108
|
+
return JSON_PARSE_FAILED;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
//const _type = Symbol("type");
|
|
113
|
+
|
|
100
114
|
var that;
|
|
101
115
|
if (!Promise.allSettled) {
|
|
102
116
|
Promise.allSettled = promises => Promise.all(promises.map((promise, i) => promise.then(value => ({
|
|
@@ -135,6 +149,7 @@ class SpiceModel {
|
|
|
135
149
|
this[_skip_cache] = ((_args4 = args) == null ? void 0 : (_args4$args = _args4.args) == null ? void 0 : _args4$args.skip_cache) || false;
|
|
136
150
|
this[_level] = ((_args5 = args) == null ? void 0 : (_args5$args = _args5.args) == null ? void 0 : _args5$args._level) || 0;
|
|
137
151
|
this[_current_path] = ((_args6 = args) == null ? void 0 : (_args6$args = _args6.args) == null ? void 0 : _args6$args._current_path) || "";
|
|
152
|
+
this[_failed_json_fields] = [];
|
|
138
153
|
this[_columns] = ((_args7 = args) == null ? void 0 : (_args7$args = _args7.args) == null ? void 0 : _args7$args._columns) || "";
|
|
139
154
|
this[_hooks] = {
|
|
140
155
|
create: {
|
|
@@ -258,13 +273,33 @@ class SpiceModel {
|
|
|
258
273
|
case Array:
|
|
259
274
|
case "array":
|
|
260
275
|
{
|
|
261
|
-
|
|
276
|
+
if (_.isArray(args.args[i])) {
|
|
277
|
+
this[i] = args.args[i];
|
|
278
|
+
} else {
|
|
279
|
+
var parsed = safeParseJsonField(args.args[i], i, args.collection || this.type);
|
|
280
|
+
if (parsed === JSON_PARSE_FAILED) {
|
|
281
|
+
this[_failed_json_fields].push(i);
|
|
282
|
+
this[i] = [];
|
|
283
|
+
} else if (parsed !== undefined) {
|
|
284
|
+
this[i] = parsed;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
262
287
|
break;
|
|
263
288
|
}
|
|
264
289
|
case Object:
|
|
265
290
|
case "object":
|
|
266
291
|
{
|
|
267
|
-
|
|
292
|
+
if (_.isObject(args.args[i])) {
|
|
293
|
+
this[i] = args.args[i];
|
|
294
|
+
} else {
|
|
295
|
+
var _parsed = safeParseJsonField(args.args[i], i, args.collection || this.type);
|
|
296
|
+
if (_parsed === JSON_PARSE_FAILED) {
|
|
297
|
+
this[_failed_json_fields].push(i);
|
|
298
|
+
this[i] = {};
|
|
299
|
+
} else if (_parsed !== undefined) {
|
|
300
|
+
this[i] = _parsed;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
268
303
|
break;
|
|
269
304
|
}
|
|
270
305
|
default:
|
|
@@ -848,6 +883,21 @@ class SpiceModel {
|
|
|
848
883
|
}
|
|
849
884
|
})();
|
|
850
885
|
}
|
|
886
|
+
get failedJsonFields() {
|
|
887
|
+
return this[_failed_json_fields] || [];
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
// Refuse writes when a schema-typed object/array field contained invalid
|
|
891
|
+
// JSON, so a parse fallback can never overwrite valid stored data.
|
|
892
|
+
assertNoFailedJsonFields(operation) {
|
|
893
|
+
var failed = this[_failed_json_fields] || [];
|
|
894
|
+
if (failed.length > 0) {
|
|
895
|
+
var err = new Error(this.type + ": cannot " + operation + " because field(s) [" + failed.join(", ") + "] contained invalid JSON. Stored data was left untouched.");
|
|
896
|
+
err.status = 400;
|
|
897
|
+
err.failed_json_fields = [...failed];
|
|
898
|
+
throw err;
|
|
899
|
+
}
|
|
900
|
+
}
|
|
851
901
|
update(args) {
|
|
852
902
|
var _this8 = this;
|
|
853
903
|
return _asyncToGenerator(function* () {
|
|
@@ -856,6 +906,7 @@ class SpiceModel {
|
|
|
856
906
|
var p = (_this8$_ctx = _this8[_ctx]) == null ? void 0 : _this8$_ctx.profiler;
|
|
857
907
|
var doUpdate = /*#__PURE__*/function () {
|
|
858
908
|
var _ref1 = _asyncToGenerator(function* () {
|
|
909
|
+
_this8.assertNoFailedJsonFields("update");
|
|
859
910
|
_this8.updated_at = new SDate().now();
|
|
860
911
|
var results = yield _this8.database.get(args.id);
|
|
861
912
|
var item_exist = yield _this8.exist(results);
|
|
@@ -926,6 +977,7 @@ class SpiceModel {
|
|
|
926
977
|
var p = (_this9$_ctx = _this9[_ctx]) == null ? void 0 : _this9$_ctx.profiler;
|
|
927
978
|
var doCreate = /*#__PURE__*/function () {
|
|
928
979
|
var _ref11 = _asyncToGenerator(function* () {
|
|
980
|
+
_this9.assertNoFailedJsonFields("create");
|
|
929
981
|
var form;
|
|
930
982
|
_this9.created_at = new SDate().now();
|
|
931
983
|
args = _.defaults(args, {
|
|
@@ -4,7 +4,6 @@ exports.__esModule = true;
|
|
|
4
4
|
exports.default = void 0;
|
|
5
5
|
var _lodash = _interopRequireDefault(require("lodash"));
|
|
6
6
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
7
|
-
function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function _interopRequireWildcard(e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (var _t in e) { "default" !== _t && {}.hasOwnProperty.call(e, _t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, _t)) && (i.get || i.set) ? o(f, _t, i) : f[_t] = e[_t]); } return f; })(e, t); }
|
|
8
7
|
function _extends() { return _extends = Object.assign ? Object.assign.bind() : function (n) { for (var e = 1; e < arguments.length; e++) { var t = arguments[e]; for (var r in t) { ({}).hasOwnProperty.call(t, r) && (n[r] = t[r]); } } return n; }, _extends.apply(null, arguments); }
|
|
9
8
|
function asyncGeneratorStep(n, t, e, r, o, a, c) { try { var i = n[a](c), u = i.value; } catch (n) { return void e(n); } i.done ? t(u) : Promise.resolve(u).then(r, o); }
|
|
10
9
|
function _asyncToGenerator(n) { return function () { var t = this, e = arguments; return new Promise(function (r, o) { var a = n.apply(t, e); function _next(n) { asyncGeneratorStep(a, r, o, _next, _throw, "next", n); } function _throw(n) { asyncGeneratorStep(a, r, o, _next, _throw, "throw", n); } _next(void 0); }); }; }
|
|
@@ -123,47 +122,65 @@ class RestHelper {
|
|
|
123
122
|
var filename, filePath;
|
|
124
123
|
if (download_type === "csv") {
|
|
125
124
|
var _ret = yield* function* () {
|
|
126
|
-
var {
|
|
127
|
-
flatten
|
|
128
|
-
} = yield Promise.resolve().then(() => _interopRequireWildcard(require("flat")));
|
|
129
125
|
var rows = Array.isArray(cleaned) ? cleaned : [cleaned];
|
|
130
126
|
var csvReady = rows.map(normalizeEmptyArraysForCsv);
|
|
131
|
-
var
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
127
|
+
var flattenToLeaves = function flattenToLeaves(value, prefix, output) {
|
|
128
|
+
if (prefix === void 0) {
|
|
129
|
+
prefix = "";
|
|
130
|
+
}
|
|
131
|
+
if (output === void 0) {
|
|
132
|
+
output = {};
|
|
133
|
+
}
|
|
134
|
+
if (Array.isArray(value)) {
|
|
135
|
+
if (value.length === 0 && prefix) {
|
|
136
|
+
output[prefix] = "";
|
|
137
|
+
return output;
|
|
138
|
+
}
|
|
139
|
+
value.forEach((item, index) => {
|
|
140
|
+
var path = prefix ? prefix + "." + index : "" + index;
|
|
141
|
+
flattenToLeaves(item, path, output);
|
|
142
|
+
});
|
|
143
|
+
return output;
|
|
144
|
+
}
|
|
145
|
+
if (value && typeof value === "object") {
|
|
146
|
+
var entries = Object.entries(value);
|
|
147
|
+
if (entries.length === 0 && prefix) {
|
|
148
|
+
output[prefix] = "";
|
|
149
|
+
return output;
|
|
150
|
+
}
|
|
151
|
+
for (var [key, item] of entries) {
|
|
152
|
+
var _path = prefix ? prefix + "." + key : key;
|
|
153
|
+
flattenToLeaves(item, _path, output);
|
|
154
|
+
}
|
|
155
|
+
return output;
|
|
156
|
+
}
|
|
157
|
+
if (prefix) {
|
|
158
|
+
output[prefix] = value;
|
|
159
|
+
}
|
|
160
|
+
return output;
|
|
161
|
+
};
|
|
162
|
+
var flatRows = csvReady.map(row => flattenToLeaves(row));
|
|
135
163
|
var fieldSet = new Set();
|
|
136
|
-
for (var
|
|
137
|
-
Object.keys(
|
|
164
|
+
for (var row of flatRows) {
|
|
165
|
+
Object.keys(row).forEach(field => fieldSet.add(field));
|
|
138
166
|
}
|
|
139
167
|
var fields = Array.from(fieldSet).sort((a, b) => a.localeCompare(b, undefined, {
|
|
140
168
|
numeric: true,
|
|
141
169
|
sensitivity: "base"
|
|
142
170
|
}));
|
|
143
|
-
|
|
144
|
-
// Filter flattened fields to only include requested columns
|
|
145
171
|
var rawUrl = ctx.originalUrl || ctx.request.url || "";
|
|
146
172
|
var urlColumnsMatch = rawUrl.match(/[?&]columns=([^&]*)/);
|
|
147
|
-
var requestedColumns = urlColumnsMatch ? decodeURIComponent(urlColumnsMatch[1]) : null;
|
|
173
|
+
var requestedColumns = urlColumnsMatch ? decodeURIComponent(urlColumnsMatch[1].replace(/\+/g, " ")) : null;
|
|
148
174
|
if (requestedColumns) {
|
|
149
|
-
var
|
|
150
|
-
var
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
var segments = spec.split(".");
|
|
159
|
-
var pattern = segments.map(s => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("(\\.\\d+)?\\.");
|
|
160
|
-
var re = new RegExp("^" + pattern + "(\\..+)?$");
|
|
161
|
-
for (var f of fields) {
|
|
162
|
-
if (re.test(f)) filtered.push(f);
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
fields = filtered.length > 0 ? filtered : fields;
|
|
175
|
+
var requestedSpecs = requestedColumns.split(",").map(column => column.trim().replace(/`/g, "")).filter(Boolean);
|
|
176
|
+
var normalizePath = field => field.split(".").filter(segment => !/^\d+$/.test(segment)).join(".");
|
|
177
|
+
fields = fields.filter(field => {
|
|
178
|
+
var normalizedField = normalizePath(field);
|
|
179
|
+
return requestedSpecs.some(requestedField => {
|
|
180
|
+
var normalizedRequestedField = normalizePath(requestedField);
|
|
181
|
+
return normalizedField === normalizedRequestedField || normalizedField.startsWith(normalizedRequestedField + ".");
|
|
182
|
+
});
|
|
183
|
+
});
|
|
167
184
|
}
|
|
168
185
|
var csv = parse(flatRows, {
|
|
169
186
|
fields,
|
|
@@ -178,7 +195,6 @@ class RestHelper {
|
|
|
178
195
|
ctx.type = "text/csv; charset=utf-8";
|
|
179
196
|
ctx.status = 200;
|
|
180
197
|
var csvStream = fs.createReadStream(filePath);
|
|
181
|
-
// Delete file after stream is finished
|
|
182
198
|
csvStream.on("close", () => {
|
|
183
199
|
fs.promises.unlink(filePath).catch(() => {});
|
|
184
200
|
});
|
package/package.json
CHANGED
package/src/models/SpiceModel.js
CHANGED
|
@@ -27,7 +27,8 @@ var SDate = require("sonover-date"),
|
|
|
27
27
|
_mapping_concurrency = Symbol(),
|
|
28
28
|
_skip_mapped_serialization = Symbol(),
|
|
29
29
|
_mapping_dept_exempt = Symbol(),
|
|
30
|
-
_current_path = Symbol()
|
|
30
|
+
_current_path = Symbol(),
|
|
31
|
+
_failed_json_fields = Symbol();
|
|
31
32
|
|
|
32
33
|
const GLOBAL_MAPPING_CONCURRENCY = Math.max(
|
|
33
34
|
1,
|
|
@@ -82,6 +83,20 @@ async function runWithConcurrency(items, concurrency, worker) {
|
|
|
82
83
|
return results;
|
|
83
84
|
}
|
|
84
85
|
|
|
86
|
+
const JSON_PARSE_FAILED = Symbol("json_parse_failed");
|
|
87
|
+
|
|
88
|
+
function safeParseJsonField(value, fieldName, modelType) {
|
|
89
|
+
if (value == null || value === "") return undefined;
|
|
90
|
+
try {
|
|
91
|
+
return JSON.parse(value);
|
|
92
|
+
} catch (e) {
|
|
93
|
+
console.warn(
|
|
94
|
+
`SpiceModel(${modelType}): failed to parse field "${fieldName}" as JSON: ${e.message}`,
|
|
95
|
+
);
|
|
96
|
+
return JSON_PARSE_FAILED;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
85
100
|
//const _type = Symbol("type");
|
|
86
101
|
|
|
87
102
|
let that;
|
|
@@ -144,6 +159,7 @@ export default class SpiceModel {
|
|
|
144
159
|
this[_skip_cache] = args?.args?.skip_cache || false;
|
|
145
160
|
this[_level] = args?.args?._level || 0;
|
|
146
161
|
this[_current_path] = args?.args?._current_path || "";
|
|
162
|
+
this[_failed_json_fields] = [];
|
|
147
163
|
this[_columns] = args?.args?._columns || "";
|
|
148
164
|
this[_hooks] = {
|
|
149
165
|
create: {
|
|
@@ -289,18 +305,40 @@ export default class SpiceModel {
|
|
|
289
305
|
}
|
|
290
306
|
case Array:
|
|
291
307
|
case "array": {
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
308
|
+
if (_.isArray(args.args[i])) {
|
|
309
|
+
this[i] = args.args[i];
|
|
310
|
+
} else {
|
|
311
|
+
const parsed = safeParseJsonField(
|
|
312
|
+
args.args[i],
|
|
313
|
+
i,
|
|
314
|
+
args.collection || this.type,
|
|
315
|
+
);
|
|
316
|
+
if (parsed === JSON_PARSE_FAILED) {
|
|
317
|
+
this[_failed_json_fields].push(i);
|
|
318
|
+
this[i] = [];
|
|
319
|
+
} else if (parsed !== undefined) {
|
|
320
|
+
this[i] = parsed;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
296
323
|
break;
|
|
297
324
|
}
|
|
298
325
|
case Object:
|
|
299
326
|
case "object": {
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
327
|
+
if (_.isObject(args.args[i])) {
|
|
328
|
+
this[i] = args.args[i];
|
|
329
|
+
} else {
|
|
330
|
+
const parsed = safeParseJsonField(
|
|
331
|
+
args.args[i],
|
|
332
|
+
i,
|
|
333
|
+
args.collection || this.type,
|
|
334
|
+
);
|
|
335
|
+
if (parsed === JSON_PARSE_FAILED) {
|
|
336
|
+
this[_failed_json_fields].push(i);
|
|
337
|
+
this[i] = {};
|
|
338
|
+
} else if (parsed !== undefined) {
|
|
339
|
+
this[i] = parsed;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
304
342
|
break;
|
|
305
343
|
}
|
|
306
344
|
default: {
|
|
@@ -941,11 +979,32 @@ export default class SpiceModel {
|
|
|
941
979
|
}
|
|
942
980
|
}
|
|
943
981
|
|
|
982
|
+
get failedJsonFields() {
|
|
983
|
+
return this[_failed_json_fields] || [];
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
// Refuse writes when a schema-typed object/array field contained invalid
|
|
987
|
+
// JSON, so a parse fallback can never overwrite valid stored data.
|
|
988
|
+
assertNoFailedJsonFields(operation) {
|
|
989
|
+
const failed = this[_failed_json_fields] || [];
|
|
990
|
+
if (failed.length > 0) {
|
|
991
|
+
const err = new Error(
|
|
992
|
+
`${this.type}: cannot ${operation} because field(s) [${failed.join(
|
|
993
|
+
", ",
|
|
994
|
+
)}] contained invalid JSON. Stored data was left untouched.`,
|
|
995
|
+
);
|
|
996
|
+
err.status = 400;
|
|
997
|
+
err.failed_json_fields = [...failed];
|
|
998
|
+
throw err;
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
|
|
944
1002
|
async update(args) {
|
|
945
1003
|
// Profiling: use track() for proper async context forking
|
|
946
1004
|
const p = this[_ctx]?.profiler;
|
|
947
1005
|
|
|
948
1006
|
const doUpdate = async () => {
|
|
1007
|
+
this.assertNoFailedJsonFields("update");
|
|
949
1008
|
this.updated_at = new SDate().now();
|
|
950
1009
|
let results = await this.database.get(args.id);
|
|
951
1010
|
let item_exist = await this.exist(results);
|
|
@@ -1025,6 +1084,7 @@ export default class SpiceModel {
|
|
|
1025
1084
|
const p = this[_ctx]?.profiler;
|
|
1026
1085
|
|
|
1027
1086
|
const doCreate = async () => {
|
|
1087
|
+
this.assertNoFailedJsonFields("create");
|
|
1028
1088
|
let form;
|
|
1029
1089
|
this.created_at = new SDate().now();
|
|
1030
1090
|
args = _.defaults(args, { id_prefix: this.type });
|
|
@@ -100,7 +100,6 @@ export default class RestHelper {
|
|
|
100
100
|
try {
|
|
101
101
|
const download_type = (ctx.request.query.format || "csv").toLowerCase();
|
|
102
102
|
const include_id = ctx.request.query.include_id;
|
|
103
|
-
|
|
104
103
|
const original = _.cloneDeep(ctx.data);
|
|
105
104
|
const trimmed = deepTrimStrings(original);
|
|
106
105
|
const cleaned = stripTopLevel(trimmed, {
|
|
@@ -108,78 +107,126 @@ export default class RestHelper {
|
|
|
108
107
|
});
|
|
109
108
|
|
|
110
109
|
let filename, filePath;
|
|
111
|
-
|
|
112
110
|
if (download_type === "csv") {
|
|
113
|
-
const { flatten } = await import("flat");
|
|
114
111
|
const rows = Array.isArray(cleaned) ? cleaned : [cleaned];
|
|
115
|
-
|
|
112
|
+
|
|
116
113
|
const csvReady = rows.map(normalizeEmptyArraysForCsv);
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
114
|
+
|
|
115
|
+
const flattenToLeaves = (value, prefix = "", output = {}) => {
|
|
116
|
+
if (Array.isArray(value)) {
|
|
117
|
+
if (value.length === 0 && prefix) {
|
|
118
|
+
output[prefix] = "";
|
|
119
|
+
return output;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
value.forEach((item, index) => {
|
|
123
|
+
const path = prefix ? `${prefix}.${index}` : `${index}`;
|
|
124
|
+
flattenToLeaves(item, path, output);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
return output;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (value && typeof value === "object") {
|
|
131
|
+
const entries = Object.entries(value);
|
|
132
|
+
|
|
133
|
+
if (entries.length === 0 && prefix) {
|
|
134
|
+
output[prefix] = "";
|
|
135
|
+
return output;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
for (const [key, item] of entries) {
|
|
139
|
+
const path = prefix ? `${prefix}.${key}` : key;
|
|
140
|
+
flattenToLeaves(item, path, output);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
return output;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (prefix) {
|
|
147
|
+
output[prefix] = value;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
return output;
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
const flatRows = csvReady.map((row) => flattenToLeaves(row));
|
|
154
|
+
|
|
121
155
|
const fieldSet = new Set();
|
|
122
|
-
|
|
123
|
-
|
|
156
|
+
|
|
157
|
+
for (const row of flatRows) {
|
|
158
|
+
Object.keys(row).forEach((field) => fieldSet.add(field));
|
|
159
|
+
}
|
|
160
|
+
|
|
124
161
|
let fields = Array.from(fieldSet).sort((a, b) =>
|
|
125
|
-
a.localeCompare(b, undefined, {
|
|
162
|
+
a.localeCompare(b, undefined, {
|
|
163
|
+
numeric: true,
|
|
164
|
+
sensitivity: "base",
|
|
165
|
+
}),
|
|
126
166
|
);
|
|
127
|
-
|
|
128
|
-
// Filter flattened fields to only include requested columns
|
|
167
|
+
|
|
129
168
|
const rawUrl = ctx.originalUrl || ctx.request.url || "";
|
|
130
169
|
const urlColumnsMatch = rawUrl.match(/[?&]columns=([^&]*)/);
|
|
170
|
+
|
|
131
171
|
const requestedColumns = urlColumnsMatch
|
|
132
|
-
? decodeURIComponent(urlColumnsMatch[1])
|
|
172
|
+
? decodeURIComponent(urlColumnsMatch[1].replace(/\+/g, " "))
|
|
133
173
|
: null;
|
|
174
|
+
|
|
134
175
|
if (requestedColumns) {
|
|
135
|
-
const
|
|
176
|
+
const requestedSpecs = requestedColumns
|
|
136
177
|
.split(",")
|
|
137
|
-
.map((
|
|
138
|
-
.filter(
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
178
|
+
.map((column) => column.trim().replace(/`/g, ""))
|
|
179
|
+
.filter(Boolean);
|
|
180
|
+
|
|
181
|
+
const normalizePath = (field) =>
|
|
182
|
+
field
|
|
183
|
+
.split(".")
|
|
184
|
+
.filter((segment) => !/^\d+$/.test(segment))
|
|
185
|
+
.join(".");
|
|
186
|
+
|
|
187
|
+
fields = fields.filter((field) => {
|
|
188
|
+
const normalizedField = normalizePath(field);
|
|
189
|
+
|
|
190
|
+
return requestedSpecs.some((requestedField) => {
|
|
191
|
+
const normalizedRequestedField = normalizePath(requestedField);
|
|
192
|
+
|
|
193
|
+
return (
|
|
194
|
+
normalizedField === normalizedRequestedField ||
|
|
195
|
+
normalizedField.startsWith(`${normalizedRequestedField}.`)
|
|
196
|
+
);
|
|
197
|
+
});
|
|
198
|
+
});
|
|
158
199
|
}
|
|
159
|
-
|
|
200
|
+
|
|
160
201
|
const csv = parse(flatRows, {
|
|
161
202
|
fields,
|
|
162
203
|
defaultValue: "",
|
|
163
204
|
excelStrings: true,
|
|
164
205
|
});
|
|
165
|
-
|
|
166
|
-
makeDirectory(
|
|
206
|
+
|
|
207
|
+
makeDirectory("./storage/exports/csv");
|
|
208
|
+
|
|
167
209
|
filename = `${RestHelper.makeid(9)}.csv`;
|
|
168
210
|
filePath = path.resolve(`./storage/exports/csv/${filename}`);
|
|
211
|
+
|
|
169
212
|
await fs.promises.writeFile(filePath, csv, "utf8");
|
|
170
|
-
|
|
171
|
-
ctx.set(
|
|
213
|
+
|
|
214
|
+
ctx.set(
|
|
215
|
+
"Content-Disposition",
|
|
216
|
+
`attachment; filename="${filename}"`,
|
|
217
|
+
);
|
|
172
218
|
ctx.type = "text/csv; charset=utf-8";
|
|
173
219
|
ctx.status = 200;
|
|
220
|
+
|
|
174
221
|
const csvStream = fs.createReadStream(filePath);
|
|
175
|
-
|
|
222
|
+
|
|
176
223
|
csvStream.on("close", () => {
|
|
177
224
|
fs.promises.unlink(filePath).catch(() => {});
|
|
178
225
|
});
|
|
226
|
+
|
|
179
227
|
ctx.body = csvStream;
|
|
180
228
|
return;
|
|
181
229
|
}
|
|
182
|
-
|
|
183
230
|
const jsonText = safeJSONStringify(cleaned, 2);
|
|
184
231
|
|
|
185
232
|
makeDirectory(`./storage/exports/json`);
|
|
@@ -202,6 +249,7 @@ export default class RestHelper {
|
|
|
202
249
|
}
|
|
203
250
|
}
|
|
204
251
|
|
|
252
|
+
|
|
205
253
|
static makeid(length) {
|
|
206
254
|
var result = "";
|
|
207
255
|
var characters =
|
|
@@ -44,6 +44,18 @@ class MockDatabase {
|
|
|
44
44
|
return updated;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Mock insert operation
|
|
49
|
+
* @param {string} id - Document ID
|
|
50
|
+
* @param {Object} data - Document data
|
|
51
|
+
* @returns {Promise<Object>} Inserted document
|
|
52
|
+
*/
|
|
53
|
+
async insert(id, data, ttl) {
|
|
54
|
+
const inserted = { ...data, id };
|
|
55
|
+
this.mockData.set(id, inserted);
|
|
56
|
+
return inserted;
|
|
57
|
+
}
|
|
58
|
+
|
|
47
59
|
/**
|
|
48
60
|
* Seed mock database with test data
|
|
49
61
|
* @param {string} id - Document ID
|
|
@@ -631,3 +631,255 @@ describe('SpiceModel - Critical Fixes for Empty/Null Values', () => {
|
|
|
631
631
|
});
|
|
632
632
|
});
|
|
633
633
|
});
|
|
634
|
+
|
|
635
|
+
describe('SpiceModel - Constructor JSON parse resilience', () => {
|
|
636
|
+
let warnSpy;
|
|
637
|
+
|
|
638
|
+
beforeEach(() => {
|
|
639
|
+
warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
|
|
640
|
+
});
|
|
641
|
+
|
|
642
|
+
afterEach(() => {
|
|
643
|
+
warnSpy.mockRestore();
|
|
644
|
+
});
|
|
645
|
+
|
|
646
|
+
test('should fall back to {} for HTML in object-typed field and finish init', () => {
|
|
647
|
+
const modifiersSpy = jest.spyOn(
|
|
648
|
+
SpiceModel.prototype,
|
|
649
|
+
'createModifiersFromProps'
|
|
650
|
+
);
|
|
651
|
+
|
|
652
|
+
const model = createTestModel({
|
|
653
|
+
type: 'application',
|
|
654
|
+
props: {
|
|
655
|
+
name: { type: String },
|
|
656
|
+
metadata: { type: Object },
|
|
657
|
+
title: { type: String },
|
|
658
|
+
},
|
|
659
|
+
args: {
|
|
660
|
+
name: 'App One',
|
|
661
|
+
metadata: '<p style="color:red">broken</p>',
|
|
662
|
+
title: 'Still set',
|
|
663
|
+
},
|
|
664
|
+
});
|
|
665
|
+
|
|
666
|
+
expect(model.metadata).toEqual({});
|
|
667
|
+
expect(model.name).toBe('App One');
|
|
668
|
+
expect(model.title).toBe('Still set');
|
|
669
|
+
expect(modifiersSpy).toHaveBeenCalled();
|
|
670
|
+
expect(warnSpy).toHaveBeenCalledWith(
|
|
671
|
+
expect.stringContaining('failed to parse field "metadata"')
|
|
672
|
+
);
|
|
673
|
+
|
|
674
|
+
modifiersSpy.mockRestore();
|
|
675
|
+
});
|
|
676
|
+
|
|
677
|
+
test('should fall back to [] for invalid JSON in array-typed field', () => {
|
|
678
|
+
const model = createTestModel({
|
|
679
|
+
type: 'addition',
|
|
680
|
+
props: {
|
|
681
|
+
name: { type: String },
|
|
682
|
+
tags: { type: Array },
|
|
683
|
+
},
|
|
684
|
+
args: {
|
|
685
|
+
name: 'Addition One',
|
|
686
|
+
tags: '<p style="">not json</p>',
|
|
687
|
+
},
|
|
688
|
+
});
|
|
689
|
+
|
|
690
|
+
expect(model.tags).toEqual([]);
|
|
691
|
+
expect(model.name).toBe('Addition One');
|
|
692
|
+
expect(warnSpy).toHaveBeenCalledWith(
|
|
693
|
+
expect.stringContaining('failed to parse field "tags"')
|
|
694
|
+
);
|
|
695
|
+
});
|
|
696
|
+
|
|
697
|
+
test('should parse valid JSON strings for object and array fields', () => {
|
|
698
|
+
const model = createTestModel({
|
|
699
|
+
type: 'application',
|
|
700
|
+
props: {
|
|
701
|
+
metadata: { type: Object },
|
|
702
|
+
tags: { type: Array },
|
|
703
|
+
},
|
|
704
|
+
args: {
|
|
705
|
+
metadata: '{"foo":"bar"}',
|
|
706
|
+
tags: '["a","b"]',
|
|
707
|
+
},
|
|
708
|
+
});
|
|
709
|
+
|
|
710
|
+
expect(model.metadata).toEqual({ foo: 'bar' });
|
|
711
|
+
expect(model.tags).toEqual(['a', 'b']);
|
|
712
|
+
expect(warnSpy).not.toHaveBeenCalledWith(
|
|
713
|
+
expect.stringContaining('failed to parse field')
|
|
714
|
+
);
|
|
715
|
+
});
|
|
716
|
+
|
|
717
|
+
test('should pass through already-parsed object and array values', () => {
|
|
718
|
+
const metadata = { foo: 'bar' };
|
|
719
|
+
const tags = ['a', 'b'];
|
|
720
|
+
|
|
721
|
+
const model = createTestModel({
|
|
722
|
+
type: 'application',
|
|
723
|
+
props: {
|
|
724
|
+
metadata: { type: Object },
|
|
725
|
+
tags: { type: Array },
|
|
726
|
+
},
|
|
727
|
+
args: {
|
|
728
|
+
metadata,
|
|
729
|
+
tags,
|
|
730
|
+
},
|
|
731
|
+
});
|
|
732
|
+
|
|
733
|
+
expect(model.metadata).toBe(metadata);
|
|
734
|
+
expect(model.tags).toBe(tags);
|
|
735
|
+
});
|
|
736
|
+
|
|
737
|
+
test('should track failed fields on the model', () => {
|
|
738
|
+
const model = createTestModel({
|
|
739
|
+
type: 'application',
|
|
740
|
+
props: {
|
|
741
|
+
metadata: { type: Object },
|
|
742
|
+
tags: { type: Array },
|
|
743
|
+
},
|
|
744
|
+
args: {
|
|
745
|
+
metadata: '<p>bad</p>',
|
|
746
|
+
tags: 'also bad',
|
|
747
|
+
},
|
|
748
|
+
});
|
|
749
|
+
|
|
750
|
+
expect(model.failedJsonFields.sort()).toEqual(['metadata', 'tags']);
|
|
751
|
+
});
|
|
752
|
+
|
|
753
|
+
test('should have no failed fields when all values parse', () => {
|
|
754
|
+
const model = createTestModel({
|
|
755
|
+
type: 'application',
|
|
756
|
+
props: {
|
|
757
|
+
metadata: { type: Object },
|
|
758
|
+
},
|
|
759
|
+
args: {
|
|
760
|
+
metadata: '{"foo":"bar"}',
|
|
761
|
+
},
|
|
762
|
+
});
|
|
763
|
+
|
|
764
|
+
expect(model.failedJsonFields).toEqual([]);
|
|
765
|
+
});
|
|
766
|
+
});
|
|
767
|
+
|
|
768
|
+
describe('SpiceModel - Write protection for failed JSON fields', () => {
|
|
769
|
+
let warnSpy;
|
|
770
|
+
|
|
771
|
+
beforeEach(() => {
|
|
772
|
+
warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
|
|
773
|
+
});
|
|
774
|
+
|
|
775
|
+
afterEach(() => {
|
|
776
|
+
warnSpy.mockRestore();
|
|
777
|
+
});
|
|
778
|
+
|
|
779
|
+
test('update should reject and preserve stored data when a field failed to parse', async () => {
|
|
780
|
+
const model = createTestModel({
|
|
781
|
+
type: 'application',
|
|
782
|
+
props: {
|
|
783
|
+
name: { type: String },
|
|
784
|
+
metadata: { type: Object },
|
|
785
|
+
},
|
|
786
|
+
args: {
|
|
787
|
+
name: 'Updated Name',
|
|
788
|
+
metadata: '<p style="color:red">broken html</p>',
|
|
789
|
+
skip_serialize: true,
|
|
790
|
+
skip_hooks: true,
|
|
791
|
+
},
|
|
792
|
+
});
|
|
793
|
+
|
|
794
|
+
const storedDoc = {
|
|
795
|
+
id: 'application-1',
|
|
796
|
+
name: 'Original Name',
|
|
797
|
+
metadata: { important: 'data' },
|
|
798
|
+
};
|
|
799
|
+
seedDatabase(model, storedDoc);
|
|
800
|
+
jest.spyOn(model.database, 'update');
|
|
801
|
+
|
|
802
|
+
await expect(
|
|
803
|
+
model.update({ id: 'application-1', skip_serialize: true, skip_hooks: true })
|
|
804
|
+
).rejects.toThrow(/cannot update.*metadata.*invalid JSON/);
|
|
805
|
+
|
|
806
|
+
// No write must reach the database; stored data stays intact
|
|
807
|
+
expect(model.database.update).not.toHaveBeenCalled();
|
|
808
|
+
const stored = await model.database.get('application-1');
|
|
809
|
+
expect(stored.metadata).toEqual({ important: 'data' });
|
|
810
|
+
});
|
|
811
|
+
|
|
812
|
+
test('update rejection should carry status 400 and the failed field names', async () => {
|
|
813
|
+
const model = createTestModel({
|
|
814
|
+
type: 'application',
|
|
815
|
+
props: {
|
|
816
|
+
metadata: { type: Object },
|
|
817
|
+
},
|
|
818
|
+
args: {
|
|
819
|
+
metadata: 'not json at all',
|
|
820
|
+
skip_serialize: true,
|
|
821
|
+
skip_hooks: true,
|
|
822
|
+
},
|
|
823
|
+
});
|
|
824
|
+
seedDatabase(model, { id: 'application-1', metadata: { keep: true } });
|
|
825
|
+
|
|
826
|
+
let caught;
|
|
827
|
+
try {
|
|
828
|
+
await model.update({ id: 'application-1', skip_serialize: true, skip_hooks: true });
|
|
829
|
+
} catch (e) {
|
|
830
|
+
caught = e;
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
expect(caught).toBeDefined();
|
|
834
|
+
expect(caught.status).toBe(400);
|
|
835
|
+
expect(caught.failed_json_fields).toEqual(['metadata']);
|
|
836
|
+
});
|
|
837
|
+
|
|
838
|
+
test('create should reject when a field failed to parse', async () => {
|
|
839
|
+
const model = createTestModel({
|
|
840
|
+
type: 'application',
|
|
841
|
+
props: {
|
|
842
|
+
metadata: { type: Object },
|
|
843
|
+
},
|
|
844
|
+
args: {
|
|
845
|
+
metadata: '<div>nope</div>',
|
|
846
|
+
skip_serialize: true,
|
|
847
|
+
skip_hooks: true,
|
|
848
|
+
},
|
|
849
|
+
});
|
|
850
|
+
jest.spyOn(model.database, 'insert');
|
|
851
|
+
|
|
852
|
+
await expect(
|
|
853
|
+
model.create({ skip_serialize: true, skip_hooks: true })
|
|
854
|
+
).rejects.toThrow(/cannot create.*metadata.*invalid JSON/);
|
|
855
|
+
expect(model.database.insert).not.toHaveBeenCalled();
|
|
856
|
+
});
|
|
857
|
+
|
|
858
|
+
test('update should proceed normally when all fields parsed', async () => {
|
|
859
|
+
const model = createTestModel({
|
|
860
|
+
type: 'application',
|
|
861
|
+
props: {
|
|
862
|
+
name: { type: String },
|
|
863
|
+
metadata: { type: Object },
|
|
864
|
+
},
|
|
865
|
+
args: {
|
|
866
|
+
name: 'New Name',
|
|
867
|
+
metadata: '{"valid":"json"}',
|
|
868
|
+
skip_serialize: true,
|
|
869
|
+
skip_hooks: true,
|
|
870
|
+
},
|
|
871
|
+
});
|
|
872
|
+
seedDatabase(model, { id: 'application-1', name: 'Old', metadata: {} });
|
|
873
|
+
// setMonitor needs the real cache global, which test mocks don't provide
|
|
874
|
+
jest.spyOn(model, 'setMonitor').mockImplementation(() => {});
|
|
875
|
+
|
|
876
|
+
const result = await model.update({
|
|
877
|
+
id: 'application-1',
|
|
878
|
+
skip_serialize: true,
|
|
879
|
+
skip_hooks: true,
|
|
880
|
+
});
|
|
881
|
+
|
|
882
|
+
expect(result.name).toBe('New Name');
|
|
883
|
+
expect(result.metadata).toEqual({ valid: 'json' });
|
|
884
|
+
});
|
|
885
|
+
});
|