prostgles-server 4.0.57 → 4.0.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/dist/DboBuilder/getColumns.d.ts.map +1 -1
  2. package/dist/DboBuilder/getColumns.js +1 -13
  3. package/dist/DboBuilder/getColumns.js.map +1 -1
  4. package/dist/DboBuilder/runSQL.js +2 -2
  5. package/dist/DboBuilder/runSQL.js.map +1 -1
  6. package/dist/Prostgles.d.ts.map +1 -1
  7. package/dist/PubSubManager/PubSubManager.d.ts +1 -1
  8. package/dist/PubSubManager/PubSubManager.d.ts.map +1 -1
  9. package/dist/PubSubManager/PubSubManager.js +14 -1
  10. package/dist/PubSubManager/PubSubManager.js.map +1 -1
  11. package/lib/DboBuilder/getColumns.ts +1 -18
  12. package/lib/DboBuilder/runSQL.ts +2 -2
  13. package/lib/Prostgles.ts +3 -3
  14. package/lib/PubSubManager/PubSubManager.ts +14 -1
  15. package/package.json +2 -2
  16. package/tests/client/PID.txt +1 -1
  17. package/tests/client/package.json +2 -2
  18. package/tests/client/tsconfig.json +2 -1
  19. package/tests/isomorphic_queries.ts +11 -9
  20. package/tests/server/DBoGenerated.d.ts +1 -1
  21. package/tests/server/dboTypeCheck.ts +3 -3
  22. package/tests/server/index.ts +7 -6
  23. package/tests/server/package-lock.json +1 -1
  24. package/tests/server/package.json +3 -3
  25. package/tests/server/publishTypeCheck.ts +1 -1
  26. package/tests/server/tsconfig.json +3 -2
  27. package/tests/test.sh +2 -0
  28. package/dist/FileManager.d.ts +0 -143
  29. package/dist/FileManager.d.ts.map +0 -1
  30. package/dist/FileManager.js +0 -646
  31. package/dist/FileManager.js.map +0 -1
  32. package/dist/TableConfig.d.ts +0 -267
  33. package/dist/TableConfig.d.ts.map +0 -1
  34. package/dist/TableConfig.js +0 -463
  35. package/dist/TableConfig.js.map +0 -1
  36. package/dist/validation.d.ts +0 -126
  37. package/dist/validation.d.ts.map +0 -1
  38. package/dist/validation.js +0 -405
  39. package/dist/validation.js.map +0 -1
  40. package/lib/AuthHandler.d.ts +0 -187
  41. package/lib/AuthHandler.d.ts.map +0 -1
  42. package/lib/AuthHandler.js +0 -478
  43. package/lib/DBEventsManager.d.ts +0 -38
  44. package/lib/DBEventsManager.d.ts.map +0 -1
  45. package/lib/DBEventsManager.js +0 -140
  46. package/lib/DBSchemaBuilder.d.ts +0 -13
  47. package/lib/DBSchemaBuilder.d.ts.map +0 -1
  48. package/lib/DBSchemaBuilder.js +0 -145
  49. package/lib/DboBuilder/QueryBuilder/Functions.d.ts +0 -64
  50. package/lib/DboBuilder/QueryBuilder/Functions.d.ts.map +0 -1
  51. package/lib/DboBuilder/QueryBuilder/Functions.js +0 -984
  52. package/lib/DboBuilder/QueryBuilder/QueryBuilder.d.ts +0 -73
  53. package/lib/DboBuilder/QueryBuilder/QueryBuilder.d.ts.map +0 -1
  54. package/lib/DboBuilder/QueryBuilder/QueryBuilder.js +0 -335
  55. package/lib/DboBuilder/QueryBuilder/makeSelectQuery.d.ts +0 -8
  56. package/lib/DboBuilder/QueryBuilder/makeSelectQuery.d.ts.map +0 -1
  57. package/lib/DboBuilder/QueryBuilder/makeSelectQuery.js +0 -227
  58. package/lib/DboBuilder/TableHandler.d.ts +0 -37
  59. package/lib/DboBuilder/TableHandler.d.ts.map +0 -1
  60. package/lib/DboBuilder/TableHandler.js +0 -213
  61. package/lib/DboBuilder/ViewHandler.d.ts +0 -119
  62. package/lib/DboBuilder/ViewHandler.d.ts.map +0 -1
  63. package/lib/DboBuilder/ViewHandler.js +0 -1023
  64. package/lib/DboBuilder/delete.d.ts +0 -6
  65. package/lib/DboBuilder/delete.d.ts.map +0 -1
  66. package/lib/DboBuilder/delete.js +0 -128
  67. package/lib/DboBuilder/find.d.ts +0 -8
  68. package/lib/DboBuilder/find.d.ts.map +0 -1
  69. package/lib/DboBuilder/find.js +0 -91
  70. package/lib/DboBuilder/getColumns.d.ts +0 -12
  71. package/lib/DboBuilder/getColumns.d.ts.map +0 -1
  72. package/lib/DboBuilder/getColumns.js +0 -92
  73. package/lib/DboBuilder/getCondition.d.ts +0 -22
  74. package/lib/DboBuilder/getCondition.d.ts.map +0 -1
  75. package/lib/DboBuilder/getCondition.js +0 -236
  76. package/lib/DboBuilder/getSubscribeRelatedTables.d.ts +0 -20
  77. package/lib/DboBuilder/getSubscribeRelatedTables.d.ts.map +0 -1
  78. package/lib/DboBuilder/getSubscribeRelatedTables.js +0 -152
  79. package/lib/DboBuilder/getTablesForSchemaPostgresSQL.d.ts +0 -3
  80. package/lib/DboBuilder/getTablesForSchemaPostgresSQL.d.ts.map +0 -1
  81. package/lib/DboBuilder/getTablesForSchemaPostgresSQL.js +0 -207
  82. package/lib/DboBuilder/insert.d.ts +0 -6
  83. package/lib/DboBuilder/insert.d.ts.map +0 -1
  84. package/lib/DboBuilder/insert.js +0 -180
  85. package/lib/DboBuilder/insertDataParse.d.ts +0 -12
  86. package/lib/DboBuilder/insertDataParse.d.ts.map +0 -1
  87. package/lib/DboBuilder/insertDataParse.js +0 -253
  88. package/lib/DboBuilder/parseUpdateRules.d.ts +0 -18
  89. package/lib/DboBuilder/parseUpdateRules.d.ts.map +0 -1
  90. package/lib/DboBuilder/parseUpdateRules.js +0 -123
  91. package/lib/DboBuilder/runSQL.d.ts +0 -7
  92. package/lib/DboBuilder/runSQL.d.ts.map +0 -1
  93. package/lib/DboBuilder/runSQL.js +0 -135
  94. package/lib/DboBuilder/subscribe.d.ts +0 -20
  95. package/lib/DboBuilder/subscribe.d.ts.map +0 -1
  96. package/lib/DboBuilder/subscribe.js +0 -90
  97. package/lib/DboBuilder/update.d.ts +0 -6
  98. package/lib/DboBuilder/update.d.ts.map +0 -1
  99. package/lib/DboBuilder/update.js +0 -151
  100. package/lib/DboBuilder/uploadFile.d.ts +0 -7
  101. package/lib/DboBuilder/uploadFile.d.ts.map +0 -1
  102. package/lib/DboBuilder/uploadFile.js +0 -53
  103. package/lib/DboBuilder.d.ts +0 -306
  104. package/lib/DboBuilder.d.ts.map +0 -1
  105. package/lib/DboBuilder.js +0 -745
  106. package/lib/Event_Trigger_Tags.d.ts +0 -4
  107. package/lib/Event_Trigger_Tags.d.ts.map +0 -1
  108. package/lib/Event_Trigger_Tags.js +0 -116
  109. package/lib/FileManager/FileManager.d.ts +0 -135
  110. package/lib/FileManager/FileManager.d.ts.map +0 -1
  111. package/lib/FileManager/FileManager.js +0 -303
  112. package/lib/FileManager/initFileManager.d.ts +0 -4
  113. package/lib/FileManager/initFileManager.d.ts.map +0 -1
  114. package/lib/FileManager/initFileManager.js +0 -231
  115. package/lib/FileManager/parseFile.d.ts +0 -15
  116. package/lib/FileManager/parseFile.d.ts.map +0 -1
  117. package/lib/FileManager/parseFile.js +0 -58
  118. package/lib/FileManager/upload.d.ts +0 -6
  119. package/lib/FileManager/upload.d.ts.map +0 -1
  120. package/lib/FileManager/upload.js +0 -98
  121. package/lib/FileManager/uploadStream.d.ts +0 -5
  122. package/lib/FileManager/uploadStream.d.ts.map +0 -1
  123. package/lib/FileManager/uploadStream.js +0 -92
  124. package/lib/Filtering.d.ts +0 -15
  125. package/lib/Filtering.d.ts.map +0 -1
  126. package/lib/Filtering.js +0 -336
  127. package/lib/JSONBValidation/validate_jsonb_schema_sql.d.ts +0 -4
  128. package/lib/JSONBValidation/validate_jsonb_schema_sql.d.ts.map +0 -1
  129. package/lib/JSONBValidation/validate_jsonb_schema_sql.js +0 -500
  130. package/lib/JSONBValidation/validation.d.ts +0 -9
  131. package/lib/JSONBValidation/validation.d.ts.map +0 -1
  132. package/lib/JSONBValidation/validation.js +0 -131
  133. package/lib/PostgresNotifListenManager.d.ts +0 -28
  134. package/lib/PostgresNotifListenManager.d.ts.map +0 -1
  135. package/lib/PostgresNotifListenManager.js +0 -134
  136. package/lib/Prostgles.d.ts +0 -289
  137. package/lib/Prostgles.d.ts.map +0 -1
  138. package/lib/Prostgles.js +0 -685
  139. package/lib/PubSubManager/PubSubManager.d.ts +0 -175
  140. package/lib/PubSubManager/PubSubManager.d.ts.map +0 -1
  141. package/lib/PubSubManager/PubSubManager.js +0 -452
  142. package/lib/PubSubManager/addSub.d.ts +0 -8
  143. package/lib/PubSubManager/addSub.d.ts.map +0 -1
  144. package/lib/PubSubManager/addSub.js +0 -166
  145. package/lib/PubSubManager/addSync.d.ts +0 -8
  146. package/lib/PubSubManager/addSync.d.ts.map +0 -1
  147. package/lib/PubSubManager/addSync.js +0 -109
  148. package/lib/PubSubManager/getInitQuery.d.ts +0 -9
  149. package/lib/PubSubManager/getInitQuery.d.ts.map +0 -1
  150. package/lib/PubSubManager/getInitQuery.js +0 -552
  151. package/lib/PubSubManager/initPubSubManager.d.ts +0 -3
  152. package/lib/PubSubManager/initPubSubManager.d.ts.map +0 -1
  153. package/lib/PubSubManager/initPubSubManager.js +0 -116
  154. package/lib/PubSubManager/notifListener.d.ts +0 -5
  155. package/lib/PubSubManager/notifListener.d.ts.map +0 -1
  156. package/lib/PubSubManager/notifListener.js +0 -100
  157. package/lib/PubSubManager/pushSubData.d.ts +0 -3
  158. package/lib/PubSubManager/pushSubData.d.ts.map +0 -1
  159. package/lib/PubSubManager/pushSubData.js +0 -51
  160. package/lib/PublishParser.d.ts +0 -284
  161. package/lib/PublishParser.d.ts.map +0 -1
  162. package/lib/PublishParser.js +0 -421
  163. package/lib/SchemaWatch.d.ts +0 -13
  164. package/lib/SchemaWatch.d.ts.map +0 -1
  165. package/lib/SchemaWatch.js +0 -35
  166. package/lib/SyncReplication.d.ts +0 -34
  167. package/lib/SyncReplication.d.ts.map +0 -1
  168. package/lib/SyncReplication.js +0 -412
  169. package/lib/TableConfig/TableConfig.d.ts +0 -284
  170. package/lib/TableConfig/TableConfig.d.ts.map +0 -1
  171. package/lib/TableConfig/TableConfig.js +0 -437
  172. package/lib/TableConfig/getColumnDefinitionQuery.d.ts +0 -27
  173. package/lib/TableConfig/getColumnDefinitionQuery.d.ts.map +0 -1
  174. package/lib/TableConfig/getColumnDefinitionQuery.js +0 -87
  175. package/lib/TableConfig/getConstraintDefinitionQueries.d.ts +0 -34
  176. package/lib/TableConfig/getConstraintDefinitionQueries.d.ts.map +0 -1
  177. package/lib/TableConfig/getConstraintDefinitionQueries.js +0 -65
  178. package/lib/TableConfig/getFutureTableSchema.d.ts +0 -15
  179. package/lib/TableConfig/getFutureTableSchema.d.ts.map +0 -1
  180. package/lib/TableConfig/getFutureTableSchema.js +0 -43
  181. package/lib/TableConfig/getTableColumnQueries.d.ts +0 -16
  182. package/lib/TableConfig/getTableColumnQueries.d.ts.map +0 -1
  183. package/lib/TableConfig/getTableColumnQueries.js +0 -107
  184. package/lib/index.d.ts +0 -5
  185. package/lib/index.d.ts.map +0 -1
  186. package/lib/index.js +0 -7
  187. package/lib/shortestPath.d.ts +0 -10
  188. package/lib/shortestPath.d.ts.map +0 -1
  189. package/lib/shortestPath.js +0 -111
  190. package/lib/utils.d.ts +0 -3
  191. package/lib/utils.d.ts.map +0 -1
  192. package/lib/utils.js +0 -12
  193. package/tests/client/index.d.ts +0 -2
  194. package/tests/client/index.d.ts.map +0 -1
  195. package/tests/client/index.js +0 -80
  196. package/tests/client/index.js.map +0 -1
  197. package/tests/client_only_queries.d.ts +0 -4
  198. package/tests/client_only_queries.d.ts.map +0 -1
  199. package/tests/client_only_queries.js +0 -282
  200. package/tests/config_test/DBoGenerated.d.ts +0 -388
  201. package/tests/config_test/index.js +0 -174
  202. package/tests/config_test/index.js.map +0 -1
  203. package/tests/isomorphic_queries.d.ts +0 -9
  204. package/tests/isomorphic_queries.d.ts.map +0 -1
  205. package/tests/isomorphic_queries.js +0 -773
  206. package/tests/manual_test/DBoGenerated.d.ts +0 -398
  207. package/tests/manual_test/index.d.ts +0 -2
  208. package/tests/manual_test/index.d.ts.map +0 -1
  209. package/tests/manual_test/index.js +0 -65
  210. package/tests/server/dboTypeCheck.d.ts +0 -2
  211. package/tests/server/dboTypeCheck.d.ts.map +0 -1
  212. package/tests/server/dboTypeCheck.js +0 -27
  213. package/tests/server/index.d.ts +0 -2
  214. package/tests/server/index.d.ts.map +0 -1
  215. package/tests/server/index.js +0 -507
  216. package/tests/server/publishTypeCheck.d.ts +0 -2
  217. package/tests/server/publishTypeCheck.d.ts.map +0 -1
  218. package/tests/server/publishTypeCheck.js +0 -130
  219. package/tests/server_only_queries.d.ts +0 -2
  220. package/tests/server_only_queries.d.ts.map +0 -1
  221. package/tests/server_only_queries.js +0 -19
@@ -1,412 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.syncData = void 0;
4
- const PubSubManager_1 = require("./PubSubManager/PubSubManager");
5
- const prostgles_types_1 = require("prostgles-types");
6
- function getNumbers(numberArr) {
7
- return numberArr.filter(v => v !== null && v !== undefined && Number.isFinite(+v));
8
- }
9
- const syncData = async (_this, sync, clientData, source) => {
10
- (0, PubSubManager_1.log)("syncData", { clientData, sync: (0, PubSubManager_1.pickKeys)(sync, ["filter", "last_synced", "lr", "is_syncing"]), source });
11
- const { socket_id, channel_name, table_name, filter, table_rules, allow_delete = false, params, synced_field, id_fields = [], batch_size, wal, throttle = 0 } = sync, socket = _this.sockets[socket_id];
12
- if (!socket) {
13
- console.error("Orphaned socket", { sync, clientData });
14
- return;
15
- }
16
- const sync_fields = [synced_field, ...id_fields.sort()], orderByAsc = sync_fields.reduce((a, v) => ({ ...a, [v]: true }), {}),
17
- // orderByDesc = sync_fields.reduce((a, v) => ({ ...a, [v]: false }), {}),
18
- // desc_params = { orderBy: [{ [synced_field]: false }].concat(id_fields.map(f => ({ [f]: false }) )) },
19
- // asc_params = { orderBy: [synced_field].concat(id_fields) },
20
- rowsIdsMatch = (a, b) => {
21
- return a && b && !id_fields.find(key => (a[key]).toString() !== (b[key]).toString());
22
- }, rowsFullyMatch = (a, b) => {
23
- return rowsIdsMatch(a, b) && a?.[synced_field].toString() === b?.[synced_field].toString();
24
- }, getServerRowInfo = async (args = {}) => {
25
- const { from_synced = null, to_synced = null, offset = 0, limit } = args;
26
- const _filter = { ...filter };
27
- if (from_synced || to_synced) {
28
- _filter[synced_field] = {
29
- ...(from_synced ? { $gte: from_synced } : {}),
30
- ...(to_synced ? { $lte: to_synced } : {})
31
- };
32
- }
33
- if (_this.dbo?.[table_name]?.find === undefined || _this?.dbo?.[table_name]?.count === undefined) {
34
- throw `dbo.${table_name}.find or .count are missing or not allowed`;
35
- }
36
- const first_rows = await _this.dbo?.[table_name]?.find?.(_filter, { orderBy: orderByAsc, select: sync_fields, limit, offset }, undefined, table_rules);
37
- const last_rows = first_rows?.slice(-1);
38
- // const last_rows = await _this?.dbo[table_name]?.find?.(_filter, { orderBy: (orderByDesc as OrderBy), select: sync_fields, limit: 1, offset: -offset || 0 }, null, table_rules);
39
- const count = await _this.dbo?.[table_name]?.count?.(_filter, undefined, undefined, table_rules);
40
- return { s_fr: first_rows?.[0] || null, s_lr: last_rows?.[0] || null, s_count: count };
41
- }, getClientRowInfo = (args = {}) => {
42
- const { from_synced = null, to_synced = null, end_offset = null } = args;
43
- const res = new Promise((resolve, reject) => {
44
- const onSyncRequest = { from_synced, to_synced, end_offset }; //, forReal: true };
45
- socket.emit(channel_name, { onSyncRequest }, (resp) => {
46
- if (resp && "onSyncRequest" in resp && resp?.onSyncRequest) {
47
- const c_fr = resp.onSyncRequest.c_fr, c_lr = resp.onSyncRequest.c_lr, c_count = resp.onSyncRequest.c_count;
48
- // console.log(onSyncRequest, { c_fr, c_lr, c_count }, socket._user);
49
- return resolve({ c_fr, c_lr, c_count });
50
- }
51
- else if (resp && "err" in resp && resp?.err) {
52
- reject(resp.err);
53
- }
54
- });
55
- });
56
- return res;
57
- }, getClientData = (from_synced = 0, offset = 0) => {
58
- return new Promise((resolve, reject) => {
59
- const onPullRequest = { from_synced: from_synced || 0, offset: offset || 0, limit: batch_size };
60
- socket.emit(channel_name, { onPullRequest }, async (resp) => {
61
- if (resp && resp.data && Array.isArray(resp.data)) {
62
- // console.log({ onPullRequest, resp }, socket._user)
63
- resolve(sortClientData(resp.data));
64
- }
65
- else {
66
- reject("unexpected onPullRequest response: " + JSON.stringify(resp));
67
- }
68
- });
69
- });
70
- function sortClientData(data) {
71
- return data.sort((a, b) => {
72
- /* Order by increasing synced and ids (sorted alphabetically) */
73
- return (+a[synced_field] - +b[synced_field]) || id_fields.sort().map(idKey => a[idKey] < b[idKey] ? -1 : a[idKey] > b[idKey] ? 1 : 0).find(v => v) || 0;
74
- });
75
- }
76
- }, getServerData = async (from_synced = 0, offset = 0) => {
77
- const _filter = {
78
- ...filter,
79
- [synced_field]: { $gte: from_synced || 0 }
80
- };
81
- if (!_this?.dbo?.[table_name]?.find)
82
- throw "_this?.dbo?.[table_name]?.find is missing";
83
- try {
84
- const res = _this?.dbo?.[table_name]?.find?.(_filter, {
85
- select: params.select,
86
- orderBy: orderByAsc,
87
- offset: offset || 0,
88
- limit: batch_size
89
- }, undefined, table_rules);
90
- if (!res)
91
- throw "_this?.dbo?.[table_name]?.find is missing";
92
- return res;
93
- }
94
- catch (e) {
95
- console.error("Sync getServerData failed: ", e);
96
- throw "INTERNAL ERROR";
97
- }
98
- }, deleteData = async (deleted) => {
99
- // console.log("deleteData deleteData deleteData " + deleted.length);
100
- if (allow_delete) {
101
- return Promise.all(deleted.map(async (d) => {
102
- const id_filter = (0, PubSubManager_1.pickKeys)(d, id_fields);
103
- try {
104
- await _this.dbo[table_name].delete(id_filter, undefined, undefined, table_rules);
105
- return 1;
106
- }
107
- catch (e) {
108
- console.error(e);
109
- }
110
- return 0;
111
- }));
112
- }
113
- else {
114
- console.warn("client tried to delete data without permission (allow_delete is false)");
115
- }
116
- return false;
117
- },
118
- /**
119
- * Upserts the given client data where synced_field is higher than on server
120
- */
121
- upsertData = (data, isExpress = false) => {
122
- // console.log("isExpress", isExpress, data);
123
- return _this.dboBuilder.getTX(async (dbTX) => {
124
- const tbl = dbTX[table_name];
125
- const existingData = await tbl.find({ $or: data.map(d => (0, PubSubManager_1.pickKeys)(d, id_fields)) }, {
126
- select: [synced_field, ...id_fields],
127
- orderBy: orderByAsc,
128
- }, undefined, table_rules);
129
- let inserts = data.filter(d => !existingData.find(ed => rowsIdsMatch(ed, d)));
130
- let updates = data.filter(d => existingData.find(ed => rowsIdsMatch(ed, d) && +ed[synced_field] < +d[synced_field]));
131
- try {
132
- if (!table_rules)
133
- throw "table_rules missing";
134
- if (table_rules.update && updates.length) {
135
- const updateData = [];
136
- await Promise.all(updates.map(upd => {
137
- const id_filter = (0, PubSubManager_1.pickKeys)(upd, id_fields);
138
- const syncSafeFilter = { $and: [id_filter, { [synced_field]: { "<": upd[synced_field] } }] };
139
- updateData.push([syncSafeFilter, (0, PubSubManager_1.omitKeys)(upd, id_fields)]);
140
- }));
141
- await tbl.updateBatch(updateData, { fixIssues: true }, table_rules);
142
- }
143
- else {
144
- updates = [];
145
- }
146
- if (table_rules.insert && inserts.length) {
147
- await tbl.insert(inserts, { fixIssues: true }, undefined, table_rules);
148
- }
149
- else {
150
- inserts = [];
151
- }
152
- return { inserts, updates };
153
- }
154
- catch (e) {
155
- console.trace(e);
156
- throw e;
157
- }
158
- }).then(({ inserts, updates }) => {
159
- (0, PubSubManager_1.log)(`upsertData: inserted( ${inserts.length} ) updated( ${updates.length} ) total( ${data.length} ) \n last insert ${JSON.stringify(inserts.at(-1))} \n last update ${JSON.stringify(updates.at(-1))}`);
160
- return { inserted: inserts.length, updated: updates.length, total: data.length };
161
- })
162
- .catch(err => {
163
- console.trace("Something went wrong with syncing to server: \n ->", err, data.length, id_fields);
164
- return Promise.reject("Something went wrong with syncing to server: ");
165
- });
166
- },
167
- /**
168
- * Pushes the given data to client
169
- * @param isSynced = true if
170
- */
171
- pushData = async (data, isSynced = false, err = null) => {
172
- return new Promise((resolve, reject) => {
173
- socket.emit(channel_name, { data, isSynced }, (resp) => {
174
- if (resp && resp.ok) {
175
- // console.log("PUSHED to client: fr/lr", data[0], data[data.length - 1]);
176
- resolve({ pushed: data?.length, resp });
177
- }
178
- else {
179
- reject(resp);
180
- console.error("Unexpected response");
181
- }
182
- });
183
- });
184
- },
185
- /**
186
- * Returns the lowest synced_field between server and client by checking client and server sync data.
187
- * If last rows don't match it will find an earlier matching last row and use that last matching from_synced
188
- * If no rows or fully synced (c_lr and s_lr match) then returns null
189
- */
190
- getLastSynced = async (clientSyncInfo) => {
191
- // Get latest row info
192
- const { c_fr, c_lr, c_count } = clientSyncInfo || await getClientRowInfo();
193
- const { s_fr, s_lr, s_count } = await getServerRowInfo();
194
- // console.log("getLastSynced", clientData, socket._user )
195
- let result = null;
196
- /* Nothing to sync */
197
- if (!c_fr && !s_fr || rowsFullyMatch(c_lr, s_lr)) { // c_count === s_count &&
198
- // sync.last_synced = null;
199
- result = null;
200
- /* Sync Everything */
201
- }
202
- else if (!rowsFullyMatch(c_fr, s_fr)) {
203
- if (c_fr && s_fr) {
204
- result = Math.min(c_fr[synced_field], s_fr[synced_field]);
205
- }
206
- else if (c_fr || s_fr) {
207
- result = (c_fr || s_fr)[synced_field];
208
- }
209
- /* Sync from last matching synced value */
210
- }
211
- else if (rowsFullyMatch(c_fr, s_fr)) {
212
- if (s_lr && c_lr) {
213
- result = Math.min(...getNumbers([c_lr[synced_field], s_lr[synced_field]]));
214
- }
215
- else {
216
- result = Math.min(...getNumbers([c_fr[synced_field], s_fr?.[synced_field]]));
217
- }
218
- const min_count = Math.min(...getNumbers([c_count, s_count]));
219
- let end_offset = 1; // Math.min(s_count, c_count) - 1;
220
- let step = 0;
221
- while (min_count > 5 && end_offset < min_count) {
222
- const { c_lr = null } = await getClientRowInfo({ from_synced: 0, to_synced: result, end_offset });
223
- // console.log("getLastSynced... end_offset > " + end_offset);
224
- let server_row;
225
- if (c_lr) {
226
- const _filter = {};
227
- sync_fields.map(key => {
228
- _filter[key] = c_lr[key];
229
- });
230
- server_row = await _this?.dbo?.[table_name]?.find?.(_filter, { select: sync_fields, limit: 1 }, undefined, table_rules);
231
- }
232
- // if(rowsFullyMatch(c_lr, s_lr)){ //c_count === s_count &&
233
- if (server_row && server_row.length) {
234
- server_row = server_row[0];
235
- result = +server_row[synced_field];
236
- end_offset = min_count;
237
- // console.log(`getLastSynced found for ${table_name} -> ${result}`);
238
- }
239
- else {
240
- end_offset += 1 + step * (step > 4 ? 2 : 1);
241
- // console.log(`getLastSynced NOT found for ${table_name} -> ${result}`);
242
- }
243
- step++;
244
- }
245
- }
246
- return result;
247
- }, updateSyncLR = (data) => {
248
- if (data.length) {
249
- const lastRow = data[data.length - 1];
250
- if (sync.lr?.[synced_field] && +sync.lr?.[synced_field] > +lastRow[synced_field]) {
251
- console.error({ syncIssue: "sync.lr[synced_field] is greater than lastRow[synced_field]" });
252
- }
253
- sync.lr = lastRow;
254
- sync.last_synced = +sync.lr?.[synced_field];
255
- }
256
- },
257
- /**
258
- * Will push pull sync between client and server from a given from_synced value
259
- */
260
- syncBatch = async (from_synced) => {
261
- let offset = 0, canContinue = true;
262
- const limit = batch_size, min_synced = from_synced || 0, max_synced = from_synced;
263
- let inserted = 0, updated = 0, pushed = 0, deleted = 0, total = 0;
264
- // console.log("syncBatch", from_synced)
265
- while (canContinue) {
266
- const cData = await getClientData(min_synced, offset);
267
- if (cData.length) {
268
- const res = await upsertData(cData);
269
- inserted += res.inserted;
270
- updated += res.updated;
271
- }
272
- let sData;
273
- try {
274
- sData = await getServerData(min_synced, offset);
275
- }
276
- catch (e) {
277
- console.trace("sync getServerData err", e);
278
- await pushData(undefined, undefined, "Internal error. Check server logs");
279
- throw " d";
280
- }
281
- // console.log("allow_delete", table_rules.delete);
282
- if (allow_delete && table_rules?.delete) {
283
- const to_delete = sData.filter(d => {
284
- !cData.find(c => rowsIdsMatch(c, d));
285
- });
286
- await Promise.all(to_delete.map(d => {
287
- deleted++;
288
- return _this.dbo[table_name].delete((0, PubSubManager_1.pickKeys)(d, id_fields), {}, undefined, table_rules);
289
- }));
290
- sData = await getServerData(min_synced, offset);
291
- }
292
- const forClient = sData.filter(s => {
293
- return !cData.find(c => rowsIdsMatch(c, s) &&
294
- +c[synced_field] >= +s[synced_field]);
295
- });
296
- if (forClient.length) {
297
- const res = await pushData(forClient.filter(d => !sync.wal || !sync.wal.isInHistory(d)));
298
- pushed += res.pushed;
299
- }
300
- if (sData.length) {
301
- updateSyncLR(sData);
302
- total += sData.length;
303
- }
304
- offset += sData.length;
305
- // canContinue = offset >= limit;
306
- canContinue = sData.length >= limit;
307
- // console.log(`sData ${sData.length} limit ${limit}`);
308
- }
309
- (0, PubSubManager_1.log)(`server.syncBatch ${table_name}: inserted( ${inserted} ) updated( ${updated} ) deleted( ${deleted} ) pushed to client( ${pushed} ) total( ${total} )`, socket._user);
310
- return true;
311
- };
312
- if (!wal) {
313
- /* Used to throttle and merge incomming updates */
314
- sync.wal = new prostgles_types_1.WAL({
315
- id_fields, synced_field, throttle, batch_size,
316
- DEBUG_MODE: _this.dboBuilder.prostgles.opts.DEBUG_MODE,
317
- onSendStart: () => {
318
- sync.is_syncing = true;
319
- },
320
- onSend: async (data) => {
321
- // console.log("WAL upsertData START", data)
322
- const res = await upsertData(data, true);
323
- // const max_incoming_synced = Math.max(...data.map(d => +d[synced_field]));
324
- // if(Number.isFinite(max_incoming_synced) && max_incoming_synced > +sync.last_synced){
325
- // sync.last_synced = max_incoming_synced;
326
- // }
327
- // console.log("WAL upsertData END")
328
- /******** */
329
- /* TO DO -> Store and push patch updates instead of full data if and where possible */
330
- /******** */
331
- // 1. Store successfully upserted wal items for a couple of seconds
332
- // 2. When pushing data to clients check if any matching wal items exist
333
- // 3. Replace text fields with matching patched data
334
- return res;
335
- },
336
- onSendEnd: (batch) => {
337
- updateSyncLR(batch);
338
- sync.is_syncing = false;
339
- // console.log("syncData from WAL.onSendEnd")
340
- /**
341
- * After all data was inserted request SyncInfo from client and sync again if necessary
342
- */
343
- _this.syncData(sync, undefined, source);
344
- },
345
- });
346
- }
347
- /* Debounce sync requests */
348
- if (!sync.wal)
349
- throw "sync.wal missing";
350
- if (!sync.wal.isSending() && sync.is_syncing) {
351
- if (!_this.syncTimeout) {
352
- _this.syncTimeout = setTimeout(() => {
353
- _this.syncTimeout = undefined;
354
- // console.log("SYNC FROM TIMEOUT")
355
- _this.syncData(sync, undefined, source);
356
- }, throttle);
357
- }
358
- // console.log("SYNC THROTTLE")
359
- return;
360
- }
361
- // console.log("syncData", clientData)
362
- /**
363
- * Express data sent from a client that has already been synced
364
- * Add to WAL manager which will sync at the end
365
- */
366
- if (clientData) {
367
- if (clientData.data && Array.isArray(clientData.data) && clientData.data.length) {
368
- if (!sync.wal)
369
- throw "sync.wal missing";
370
- sync.wal.addData(clientData.data.map(d => ({ current: d })));
371
- return;
372
- // await upsertData(clientData.data, true);
373
- /* Not expecting this anymore. use normal db.table.delete channel */
374
- }
375
- else if (clientData.deleted && Array.isArray(clientData.deleted) && clientData.deleted.length) {
376
- await deleteData(clientData.deleted);
377
- }
378
- }
379
- else {
380
- // do nothing
381
- }
382
- if (sync.wal.isSending())
383
- return;
384
- sync.is_syncing = true;
385
- // from synced does not make sense. It should be sync.lr only!!!
386
- let from_synced = null;
387
- /** Sync was already synced */
388
- if (sync.lr) {
389
- const { s_lr } = await getServerRowInfo();
390
- /* Make sure trigger is not firing on freshly synced data */
391
- if (!rowsFullyMatch(sync.lr, s_lr)) {
392
- from_synced = sync.last_synced;
393
- }
394
- else {
395
- // console.log("rowsFullyMatch")
396
- }
397
- // console.log(table_name, sync.lr[synced_field])
398
- }
399
- else {
400
- from_synced = await getLastSynced(clientData);
401
- }
402
- if (from_synced !== null) {
403
- await syncBatch(from_synced);
404
- }
405
- else {
406
- // console.log("from_synced is null")
407
- }
408
- await pushData([], true);
409
- sync.is_syncing = false;
410
- // console.log(`Finished sync for ${table_name}`, socket._user);
411
- };
412
- exports.syncData = syncData;
@@ -1,284 +0,0 @@
1
- import { AnyObject, TableInfo, ALLOWED_EXTENSION, ALLOWED_CONTENT_TYPE, JSONB, ColumnInfo } from "prostgles-types";
2
- import { JoinInfo } from "../DboBuilder";
3
- import { DB, DBHandlerServer, Prostgles } from "../Prostgles";
4
- type ColExtraInfo = {
5
- min?: string | number;
6
- max?: string | number;
7
- hint?: string;
8
- };
9
- export type I18N_Config<LANG_IDS> = {
10
- [lang_id in keyof LANG_IDS]: string;
11
- };
12
- export declare const parseI18N: <LANG_IDS, Def extends string>(params: {
13
- config?: string | I18N_Config<LANG_IDS>;
14
- lang?: string | keyof LANG_IDS;
15
- defaultLang: string | keyof LANG_IDS;
16
- defaultValue: Def;
17
- }) => string | Def;
18
- type BaseTableDefinition<LANG_IDS = AnyObject> = {
19
- info?: {
20
- label?: string | I18N_Config<LANG_IDS>;
21
- };
22
- dropIfExistsCascade?: boolean;
23
- dropIfExists?: boolean;
24
- triggers?: {
25
- [triggerName: string]: {
26
- /**
27
- * Use "before" when you need to change the data before the action
28
- */
29
- type: "before" | "after" | "instead of";
30
- actions: ("insert" | "update" | "delete")[];
31
- forEach: "statement" | "row";
32
- /**
33
- * @example
34
- * DECLARE
35
- x_rec record;
36
- BEGIN
37
- raise notice '=operation: % =', TG_OP;
38
- IF (TG_OP = 'UPDATE' OR TG_OP = 'DELETE') THEN
39
- FOR x_rec IN SELECT * FROM old_table LOOP
40
- raise notice 'OLD: %', x_rec;
41
- END loop;
42
- END IF;
43
- IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
44
- FOR x_rec IN SELECT * FROM new_table LOOP
45
- raise notice 'NEW: %', x_rec;
46
- END loop;
47
- END IF;
48
-
49
- RETURN NULL;
50
- END;
51
- */
52
- query: string;
53
- };
54
- };
55
- };
56
- type LookupTableDefinition<LANG_IDS> = {
57
- isLookupTable: {
58
- values: {
59
- [id_value: string]: {} | {
60
- [lang_id in keyof LANG_IDS]: string;
61
- };
62
- };
63
- };
64
- };
65
- export type BaseColumn<LANG_IDS> = {
66
- /**
67
- * Will add these values to .getColumns() result
68
- */
69
- info?: ColExtraInfo;
70
- label?: string | Partial<{
71
- [lang_id in keyof LANG_IDS]: string;
72
- }>;
73
- };
74
- type SQLDefColumn = {
75
- /**
76
- * Raw sql statement used in creating/adding column
77
- */
78
- sqlDefinition?: string;
79
- };
80
- export type BaseColumnTypes = {
81
- defaultValue?: any;
82
- nullable?: boolean;
83
- };
84
- type TextColumn = BaseColumnTypes & {
85
- isText: true;
86
- /**
87
- * Value will be trimmed before update/insert
88
- */
89
- trimmed?: boolean;
90
- /**
91
- * Value will be lower cased before update/insert
92
- */
93
- lowerCased?: boolean;
94
- };
95
- export type JSONBColumnDef = (BaseColumnTypes & {}) & ({
96
- jsonbSchema: JSONB.JSONBSchema;
97
- jsonbSchemaType?: undefined;
98
- } | {
99
- jsonbSchema?: undefined;
100
- jsonbSchemaType: JSONB.ObjectType["type"];
101
- });
102
- /**
103
- * Allows referencing media to this table.
104
- * Requires this table to have a primary key AND a valid fileTable config
105
- */
106
- type MediaColumn = ({
107
- name: string;
108
- label?: string;
109
- files: "one" | "many";
110
- } & ({
111
- /**
112
- * https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes/accept
113
- */
114
- allowedContentType?: Record<Partial<("audio/*" | "video/*" | "image/*" | "text/*" | ALLOWED_CONTENT_TYPE)>, 1>;
115
- } | {
116
- allowedExtensions?: Record<Partial<ALLOWED_EXTENSION>, 1>;
117
- }));
118
- type ReferencedColumn = {
119
- /**
120
- * Will create a lookup table that this column will reference
121
- */
122
- references?: BaseColumnTypes & {
123
- tableName: string;
124
- /**
125
- * Defaults to id
126
- */
127
- columnName?: string;
128
- };
129
- };
130
- type JoinDef = {
131
- sourceTable: string;
132
- targetTable: string;
133
- on: JoinInfo["paths"][number]["on"];
134
- };
135
- /**
136
- * Used in specifying a join path to a table. This column name can then be used in select
137
- */
138
- type NamedJoinColumn = {
139
- label?: string;
140
- joinDef: JoinDef[];
141
- };
142
- type Enum<T extends string | number = any> = {
143
- enum: T[] | readonly T[];
144
- nullable?: boolean;
145
- defaultValue?: T;
146
- };
147
- export type ColumnConfig<LANG_IDS = {
148
- en: 1;
149
- }> = string | StrictUnion<NamedJoinColumn | MediaColumn | (BaseColumn<LANG_IDS> & (SQLDefColumn | ReferencedColumn | TextColumn | JSONBColumnDef | Enum))>;
150
- export type ColumnConfigs<LANG_IDS = {
151
- en: 1;
152
- }> = {
153
- sql: string | BaseColumn<LANG_IDS> & SQLDefColumn;
154
- join: BaseColumn<LANG_IDS> & NamedJoinColumn;
155
- media: BaseColumn<LANG_IDS> & MediaColumn;
156
- referenced: BaseColumn<LANG_IDS> & ReferencedColumn;
157
- text: BaseColumn<LANG_IDS> & TextColumn;
158
- jsonb: BaseColumn<LANG_IDS> & JSONBColumnDef;
159
- enum: BaseColumn<LANG_IDS> & Enum;
160
- };
161
- type UnionKeys<T> = T extends T ? keyof T : never;
162
- type StrictUnionHelper<T, TAll> = T extends any ? T & Partial<Record<Exclude<UnionKeys<TAll>, keyof T>, never>> : never;
163
- export type StrictUnion<T> = StrictUnionHelper<T, T>;
164
- export declare const CONSTRAINT_TYPES: readonly ["PRIMARY KEY", "UNIQUE", "CHECK"];
165
- export type TableDefinition<LANG_IDS> = {
166
- columns?: {
167
- [column_name: string]: ColumnConfig<LANG_IDS>;
168
- };
169
- constraints?: string[] | {
170
- [constraint_name: string]: string | {
171
- type: typeof CONSTRAINT_TYPES[number];
172
- dropIfExists?: boolean;
173
- /**
174
- * E.g.:
175
- * colname
176
- * col1, col2
177
- * col1 > col3
178
- */
179
- content: string;
180
- };
181
- };
182
- /**
183
- * Similar to unique constraints but expressions are allowed inside definition
184
- */
185
- replaceUniqueIndexes?: boolean;
186
- indexes?: {
187
- [index_name: string]: {
188
- /**
189
- * If true then will drop any existing index with this name
190
- * Overrides replaceUniqueIndexes
191
- */
192
- replace?: boolean;
193
- /**
194
- * Causes the system to check for duplicate values in the table when the index is created (if data already exist) and each time data is added.
195
- * Attempts to insert or update data which would result in duplicate entries will generate an error.
196
- */
197
- unique?: boolean;
198
- /**
199
- * When this option is used, PostgreSQL will build the index without taking any locks that prevent
200
- * concurrent inserts, updates, or deletes on the table; whereas a standard index build locks out writes (but not reads) on the table until it's done.
201
- * There are several caveats to be aware of when using this option — see Building Indexes Concurrently.
202
- */
203
- concurrently?: boolean;
204
- /**
205
- * Table name
206
- */
207
- /**
208
- * Column list
209
- * @example: col1, col2
210
- */
211
- columns: string;
212
- /**
213
- * Where clause without the "where"
214
- * Used to create a partial index. A partial index is an index that contains entries for only a portion of a table
215
- * Another possible application is to use WHERE with UNIQUE to enforce uniqueness over a subset of a table
216
- */
217
- where?: string;
218
- /**
219
- * The name of the index method to be used.
220
- * Choices are btree, hash, gist, and gin. The default method is btree.
221
- */
222
- using?: "btree" | "hash" | "gist" | "gin";
223
- };
224
- };
225
- };
226
- /**
227
- * Helper utility to create lookup tables for TEXT columns
228
- */
229
- export type TableConfig<LANG_IDS = {
230
- en: 1;
231
- }> = {
232
- [table_name: string]: BaseTableDefinition<LANG_IDS> & (TableDefinition<LANG_IDS> | LookupTableDefinition<LANG_IDS>);
233
- };
234
- /**
235
- * Will be run between initSQL and fileTable
236
- */
237
- export default class TableConfigurator<LANG_IDS = {
238
- en: 1;
239
- }> {
240
- instanceId: number;
241
- config?: TableConfig<LANG_IDS>;
242
- get dbo(): DBHandlerServer;
243
- get db(): DB;
244
- prostgles: Prostgles;
245
- constructor(prostgles: Prostgles);
246
- getColumnConfig: (tableName: string, colName: string) => ColumnConfig | undefined;
247
- getTableInfo: (params: {
248
- tableName: string;
249
- lang?: string;
250
- }) => TableInfo["info"] | undefined;
251
- getColInfo: (params: {
252
- col: string;
253
- table: string;
254
- lang?: string;
255
- }) => (ColExtraInfo & {
256
- label?: string;
257
- } & Pick<ColumnInfo, "jsonbSchema">) | undefined;
258
- checkColVal: (params: {
259
- col: string;
260
- table: string;
261
- value: any;
262
- }) => void;
263
- getJoinInfo: (sourceTable: string, targetTable: string) => JoinInfo | undefined;
264
- prevInitQueryHistory?: string[];
265
- initialising: boolean;
266
- init(): Promise<never>;
267
- log: (...args: any[]) => void;
268
- }
269
- type PGIndex = {
270
- schemaname: string;
271
- indexname: string;
272
- indexdef: string;
273
- escaped_identifier: string;
274
- type: string;
275
- owner: string;
276
- tablename: string;
277
- persistence: string;
278
- access_method: string;
279
- size: string;
280
- description: string | null;
281
- };
282
- export declare const getIndexes: (db: DB, tableName: string, schema: string) => Promise<PGIndex[]>;
283
- export {};
284
- //# sourceMappingURL=TableConfig.d.ts.map