retold-facto 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/.claude/launch.json +11 -0
  2. package/.dockerignore +8 -0
  3. package/.quackage.json +19 -0
  4. package/Dockerfile +26 -0
  5. package/bin/retold-facto.js +909 -0
  6. package/examples/facto-government-data.sqlite +0 -0
  7. package/examples/government-data-catalog.json +137 -0
  8. package/examples/government-data-loader.js +1432 -0
  9. package/package.json +91 -0
  10. package/scripts/facto-download.js +425 -0
  11. package/source/Retold-Facto.js +1042 -0
  12. package/source/services/Retold-Facto-BeaconProvider.js +511 -0
  13. package/source/services/Retold-Facto-CatalogManager.js +1252 -0
  14. package/source/services/Retold-Facto-DataLakeService.js +1642 -0
  15. package/source/services/Retold-Facto-DatasetManager.js +417 -0
  16. package/source/services/Retold-Facto-IngestEngine.js +1315 -0
  17. package/source/services/Retold-Facto-ProjectionEngine.js +3960 -0
  18. package/source/services/Retold-Facto-RecordManager.js +360 -0
  19. package/source/services/Retold-Facto-SchemaManager.js +1110 -0
  20. package/source/services/Retold-Facto-SourceFolderScanner.js +2243 -0
  21. package/source/services/Retold-Facto-SourceManager.js +730 -0
  22. package/source/services/Retold-Facto-StoreConnectionManager.js +441 -0
  23. package/source/services/Retold-Facto-ThroughputMonitor.js +478 -0
  24. package/source/services/web-app/codemirror-entry.js +7 -0
  25. package/source/services/web-app/pict-app/Pict-Application-Facto-Configuration.json +9 -0
  26. package/source/services/web-app/pict-app/Pict-Application-Facto.js +70 -0
  27. package/source/services/web-app/pict-app/Pict-Facto-Bundle.js +11 -0
  28. package/source/services/web-app/pict-app/providers/Pict-Provider-Facto-UI.js +66 -0
  29. package/source/services/web-app/pict-app/providers/Pict-Provider-Facto.js +69 -0
  30. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Catalog.js +93 -0
  31. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Connections.js +42 -0
  32. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Datasets.js +605 -0
  33. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Projections.js +188 -0
  34. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Scanner.js +80 -0
  35. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Schema.js +116 -0
  36. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Sources.js +104 -0
  37. package/source/services/web-app/pict-app/views/PictView-Facto-Catalog.js +526 -0
  38. package/source/services/web-app/pict-app/views/PictView-Facto-Datasets.js +173 -0
  39. package/source/services/web-app/pict-app/views/PictView-Facto-Ingest.js +259 -0
  40. package/source/services/web-app/pict-app/views/PictView-Facto-Layout.js +191 -0
  41. package/source/services/web-app/pict-app/views/PictView-Facto-Projections.js +231 -0
  42. package/source/services/web-app/pict-app/views/PictView-Facto-Records.js +326 -0
  43. package/source/services/web-app/pict-app/views/PictView-Facto-Scanner.js +624 -0
  44. package/source/services/web-app/pict-app/views/PictView-Facto-Sources.js +201 -0
  45. package/source/services/web-app/pict-app/views/PictView-Facto-Throughput.js +456 -0
  46. package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full-Configuration.json +14 -0
  47. package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full.js +391 -0
  48. package/source/services/web-app/pict-app-full/providers/PictRouter-Facto-Configuration.json +56 -0
  49. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-BottomBar.js +68 -0
  50. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Connections.js +340 -0
  51. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboard.js +149 -0
  52. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboards.js +819 -0
  53. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Datasets.js +178 -0
  54. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-IngestJobs.js +99 -0
  55. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Layout.js +62 -0
  56. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-MappingEditor.js +158 -0
  57. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-ProjectionDetail.js +1120 -0
  58. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Projections.js +172 -0
  59. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-QueryPanel.js +119 -0
  60. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-RecordViewer.js +663 -0
  61. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Records.js +648 -0
  62. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Scanner.js +1017 -0
  63. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDetail.js +1404 -0
  64. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDocEditor.js +1036 -0
  65. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaEditor.js +636 -0
  66. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaResearch.js +357 -0
  67. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceDetail.js +822 -0
  68. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceEditor.js +1036 -0
  69. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceResearch.js +487 -0
  70. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Sources.js +165 -0
  71. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Throughput.js +439 -0
  72. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-TopBar.js +335 -0
  73. package/source/services/web-app/pict-app-full/views/projections/Facto-Projections-Constants.js +71 -0
  74. package/source/services/web-app/web/chart.min.js +20 -0
  75. package/source/services/web-app/web/codemirror-bundle.js +30099 -0
  76. package/source/services/web-app/web/css/facto-themes.css +467 -0
  77. package/source/services/web-app/web/css/facto.css +502 -0
  78. package/source/services/web-app/web/index.html +28 -0
  79. package/source/services/web-app/web/retold-facto.js +12138 -0
  80. package/source/services/web-app/web/retold-facto.js.map +1 -0
  81. package/source/services/web-app/web/retold-facto.min.js +2 -0
  82. package/source/services/web-app/web/retold-facto.min.js.map +1 -0
  83. package/source/services/web-app/web/simple/index.html +17 -0
  84. package/test/Facto_Browser_Integration_tests.js +798 -0
  85. package/test/RetoldFacto_tests.js +4117 -0
  86. package/test/fixtures/weather-readings.csv +17 -0
  87. package/test/fixtures/weather-stations.csv +9 -0
  88. package/test/model/MeadowModel-Extended.json +8497 -0
  89. package/test/model/MeadowModel-PICT.json +1 -0
  90. package/test/model/MeadowModel.json +1355 -0
  91. package/test/model/ddl/Facto.ddl +225 -0
  92. package/test/model/fable-configuration.json +14 -0
@@ -0,0 +1,730 @@
1
+ /**
2
+ * Retold Facto - Source Manager Service
3
+ *
4
+ * Manages data sources and their supporting documentation.
5
+ * Provides endpoints for source CRUD beyond the auto-generated Meadow endpoints,
6
+ * including documentation management and active/inactive toggling.
7
+ *
8
+ * @author Steven Velozo <steven@velozo.com>
9
+ */
10
+ const libFableServiceProviderBase = require('fable-serviceproviderbase');
11
+
12
+ const defaultSourceManagerOptions = (
13
+ {
14
+ RoutePrefix: '/facto'
15
+ });
16
+
17
+ class RetoldFactoSourceManager extends libFableServiceProviderBase
18
+ {
19
+ constructor(pFable, pOptions, pServiceHash)
20
+ {
21
+ let tmpOptions = Object.assign({}, defaultSourceManagerOptions, pOptions);
22
+ super(pFable, tmpOptions, pServiceHash);
23
+
24
+ this.serviceType = 'RetoldFactoSourceManager';
25
+ }
26
+
27
+ /**
28
+ * Connect REST API routes for source management.
29
+ *
30
+ * @param {object} pOratorServiceServer - The Orator service server instance
31
+ */
32
+ connectRoutes(pOratorServiceServer)
33
+ {
34
+ let tmpRoutePrefix = this.options.RoutePrefix;
35
+
36
+ // GET /facto/source/by-hash/:Hash -- look up a source by its human-readable Hash
37
+ pOratorServiceServer.doGet(`${tmpRoutePrefix}/source/by-hash/:Hash`,
38
+ (pRequest, pResponse, fNext) =>
39
+ {
40
+ let tmpHash = pRequest.params.Hash;
41
+ if (!tmpHash)
42
+ {
43
+ pResponse.send({ Error: 'Hash parameter is required' });
44
+ return fNext();
45
+ }
46
+
47
+ if (!this.fable.DAL || !this.fable.DAL.Source)
48
+ {
49
+ pResponse.send({ Error: 'Source DAL not initialized' });
50
+ return fNext();
51
+ }
52
+
53
+ let tmpQuery = this.fable.DAL.Source.query.clone()
54
+ .addFilter('Hash', tmpHash)
55
+ .addFilter('Deleted', 0);
56
+
57
+ this.fable.DAL.Source.doReads(tmpQuery,
58
+ (pError, pQuery, pRecords) =>
59
+ {
60
+ if (pError)
61
+ {
62
+ pResponse.send({ Error: pError.message || pError });
63
+ return fNext();
64
+ }
65
+ if (!pRecords || pRecords.length === 0)
66
+ {
67
+ pResponse.send({ Error: `No source found with Hash "${tmpHash}"` });
68
+ return fNext();
69
+ }
70
+ pResponse.send({ Source: pRecords[0] });
71
+ return fNext();
72
+ });
73
+ });
74
+
75
+ // GET /facto/sources/active -- list all active (non-deleted) sources
76
+ pOratorServiceServer.doGet(`${tmpRoutePrefix}/sources/active`,
77
+ (pRequest, pResponse, fNext) =>
78
+ {
79
+ if (!this.fable.DAL || !this.fable.DAL.Source)
80
+ {
81
+ pResponse.send({ Error: 'Source DAL not initialized', Sources: [] });
82
+ return fNext();
83
+ }
84
+
85
+ let tmpQuery = this.fable.DAL.Source.query.clone()
86
+ .addFilter('Deleted', 0)
87
+ .addFilter('Active', 1);
88
+
89
+ this.fable.DAL.Source.doReads(tmpQuery,
90
+ (pError, pQuery, pRecords) =>
91
+ {
92
+ if (pError)
93
+ {
94
+ this.fable.log.error(`SourceManager error listing active sources: ${pError}`);
95
+ pResponse.send({ Error: pError.message || pError, Sources: [] });
96
+ return fNext();
97
+ }
98
+ pResponse.send({ Active: true, Count: pRecords.length, Sources: pRecords });
99
+ return fNext();
100
+ });
101
+ });
102
+
103
+ // PUT /facto/source/:IDSource/activate -- mark a source as active
104
+ pOratorServiceServer.doPut(`${tmpRoutePrefix}/source/:IDSource/activate`,
105
+ (pRequest, pResponse, fNext) =>
106
+ {
107
+ let tmpIDSource = parseInt(pRequest.params.IDSource, 10);
108
+ if (isNaN(tmpIDSource) || tmpIDSource < 1)
109
+ {
110
+ pResponse.send({ Error: 'Invalid IDSource parameter' });
111
+ return fNext();
112
+ }
113
+
114
+ if (!this.fable.DAL || !this.fable.DAL.Source)
115
+ {
116
+ pResponse.send({ Error: 'Source DAL not initialized' });
117
+ return fNext();
118
+ }
119
+
120
+ let tmpQuery = this.fable.DAL.Source.query.clone()
121
+ .addRecord({ IDSource: tmpIDSource, Active: 1 });
122
+
123
+ this.fable.DAL.Source.doUpdate(tmpQuery,
124
+ (pError, pQuery, pQueryRead, pRecord) =>
125
+ {
126
+ if (pError)
127
+ {
128
+ this.fable.log.error(`SourceManager error activating source ${tmpIDSource}: ${pError}`);
129
+ pResponse.send({ Error: pError.message || pError });
130
+ return fNext();
131
+ }
132
+ pResponse.send({ Success: true, Source: pRecord });
133
+ return fNext();
134
+ });
135
+ });
136
+
137
+ // PUT /facto/source/:IDSource/deactivate -- mark a source as inactive
138
+ pOratorServiceServer.doPut(`${tmpRoutePrefix}/source/:IDSource/deactivate`,
139
+ (pRequest, pResponse, fNext) =>
140
+ {
141
+ let tmpIDSource = parseInt(pRequest.params.IDSource, 10);
142
+ if (isNaN(tmpIDSource) || tmpIDSource < 1)
143
+ {
144
+ pResponse.send({ Error: 'Invalid IDSource parameter' });
145
+ return fNext();
146
+ }
147
+
148
+ if (!this.fable.DAL || !this.fable.DAL.Source)
149
+ {
150
+ pResponse.send({ Error: 'Source DAL not initialized' });
151
+ return fNext();
152
+ }
153
+
154
+ let tmpQuery = this.fable.DAL.Source.query.clone()
155
+ .addRecord({ IDSource: tmpIDSource, Active: 0 });
156
+
157
+ this.fable.DAL.Source.doUpdate(tmpQuery,
158
+ (pError, pQuery, pQueryRead, pRecord) =>
159
+ {
160
+ if (pError)
161
+ {
162
+ this.fable.log.error(`SourceManager error deactivating source ${tmpIDSource}: ${pError}`);
163
+ pResponse.send({ Error: pError.message || pError });
164
+ return fNext();
165
+ }
166
+ pResponse.send({ Success: true, Source: pRecord });
167
+ return fNext();
168
+ });
169
+ });
170
+
171
+ // GET /facto/source/:IDSource/documentation -- list documentation for a source
172
+ pOratorServiceServer.doGet(`${tmpRoutePrefix}/source/:IDSource/documentation`,
173
+ (pRequest, pResponse, fNext) =>
174
+ {
175
+ let tmpIDSource = parseInt(pRequest.params.IDSource, 10);
176
+ if (isNaN(tmpIDSource) || tmpIDSource < 1)
177
+ {
178
+ pResponse.send({ Error: 'Invalid IDSource parameter', Documentation: [] });
179
+ return fNext();
180
+ }
181
+
182
+ if (!this.fable.DAL || !this.fable.DAL.SourceDocumentation)
183
+ {
184
+ pResponse.send({ Error: 'SourceDocumentation DAL not initialized', Documentation: [] });
185
+ return fNext();
186
+ }
187
+
188
+ let tmpQuery = this.fable.DAL.SourceDocumentation.query.clone()
189
+ .addFilter('IDSource', tmpIDSource)
190
+ .addFilter('Deleted', 0);
191
+
192
+ this.fable.DAL.SourceDocumentation.doReads(tmpQuery,
193
+ (pError, pQuery, pRecords) =>
194
+ {
195
+ if (pError)
196
+ {
197
+ this.fable.log.error(`SourceManager error listing documentation for source ${tmpIDSource}: ${pError}`);
198
+ pResponse.send({ Error: pError.message || pError, Documentation: [] });
199
+ return fNext();
200
+ }
201
+ pResponse.send({ IDSource: tmpIDSource, Count: pRecords.length, Documentation: pRecords });
202
+ return fNext();
203
+ });
204
+ });
205
+
206
+ // GET /facto/source/:IDSource/summary -- get a source with its stats
207
+ pOratorServiceServer.doGet(`${tmpRoutePrefix}/source/:IDSource/summary`,
208
+ (pRequest, pResponse, fNext) =>
209
+ {
210
+ let tmpIDSource = parseInt(pRequest.params.IDSource, 10);
211
+ if (isNaN(tmpIDSource) || tmpIDSource < 1)
212
+ {
213
+ pResponse.send({ Error: 'Invalid IDSource parameter' });
214
+ return fNext();
215
+ }
216
+
217
+ if (!this.fable.DAL || !this.fable.DAL.Source)
218
+ {
219
+ pResponse.send({ Error: 'Source DAL not initialized' });
220
+ return fNext();
221
+ }
222
+
223
+ let tmpAnticipate = this.fable.newAnticipate();
224
+ let tmpResult = { IDSource: tmpIDSource, Source: false, RecordCount: 0, DatasetCount: 0, DocumentationCount: 0 };
225
+
226
+ // Load the source record
227
+ tmpAnticipate.anticipate(
228
+ (fStep) =>
229
+ {
230
+ let tmpQuery = this.fable.DAL.Source.query.clone()
231
+ .addFilter('IDSource', tmpIDSource);
232
+ this.fable.DAL.Source.doRead(tmpQuery,
233
+ (pError, pQuery, pRecord) =>
234
+ {
235
+ if (!pError && pRecord)
236
+ {
237
+ tmpResult.Source = pRecord;
238
+ }
239
+ return fStep();
240
+ });
241
+ });
242
+
243
+ // Count records from this source
244
+ tmpAnticipate.anticipate(
245
+ (fStep) =>
246
+ {
247
+ if (!this.fable.DAL.Record)
248
+ {
249
+ return fStep();
250
+ }
251
+ let tmpQuery = this.fable.DAL.Record.query.clone()
252
+ .addFilter('IDSource', tmpIDSource)
253
+ .addFilter('Deleted', 0);
254
+ this.fable.DAL.Record.doCount(tmpQuery,
255
+ (pError, pQuery, pCount) =>
256
+ {
257
+ if (!pError)
258
+ {
259
+ tmpResult.RecordCount = pCount;
260
+ }
261
+ return fStep();
262
+ });
263
+ });
264
+
265
+ // Count datasets linked to this source
266
+ tmpAnticipate.anticipate(
267
+ (fStep) =>
268
+ {
269
+ if (!this.fable.DAL.DatasetSource)
270
+ {
271
+ return fStep();
272
+ }
273
+ let tmpQuery = this.fable.DAL.DatasetSource.query.clone()
274
+ .addFilter('IDSource', tmpIDSource)
275
+ .addFilter('Deleted', 0);
276
+ this.fable.DAL.DatasetSource.doCount(tmpQuery,
277
+ (pError, pQuery, pCount) =>
278
+ {
279
+ if (!pError)
280
+ {
281
+ tmpResult.DatasetCount = pCount;
282
+ }
283
+ return fStep();
284
+ });
285
+ });
286
+
287
+ // Count documentation entries
288
+ tmpAnticipate.anticipate(
289
+ (fStep) =>
290
+ {
291
+ if (!this.fable.DAL.SourceDocumentation)
292
+ {
293
+ return fStep();
294
+ }
295
+ let tmpQuery = this.fable.DAL.SourceDocumentation.query.clone()
296
+ .addFilter('IDSource', tmpIDSource)
297
+ .addFilter('Deleted', 0);
298
+ this.fable.DAL.SourceDocumentation.doCount(tmpQuery,
299
+ (pError, pQuery, pCount) =>
300
+ {
301
+ if (!pError)
302
+ {
303
+ tmpResult.DocumentationCount = pCount;
304
+ }
305
+ return fStep();
306
+ });
307
+ });
308
+
309
+ tmpAnticipate.wait(
310
+ (pError) =>
311
+ {
312
+ if (pError)
313
+ {
314
+ pResponse.send({ Error: pError.message || pError });
315
+ return fNext();
316
+ }
317
+ pResponse.send(tmpResult);
318
+ return fNext();
319
+ });
320
+ });
321
+
322
+ // POST /facto/source/:IDSource/documentation -- create a documentation record
323
+ pOratorServiceServer.doPost(`${tmpRoutePrefix}/source/:IDSource/documentation`,
324
+ (pRequest, pResponse, fNext) =>
325
+ {
326
+ let tmpIDSource = parseInt(pRequest.params.IDSource, 10);
327
+ if (isNaN(tmpIDSource) || tmpIDSource < 1)
328
+ {
329
+ pResponse.send({ Error: 'Invalid IDSource parameter' });
330
+ return fNext();
331
+ }
332
+
333
+ if (!this.fable.DAL || !this.fable.DAL.SourceDocumentation)
334
+ {
335
+ pResponse.send({ Error: 'SourceDocumentation DAL not initialized' });
336
+ return fNext();
337
+ }
338
+
339
+ let tmpBody = pRequest.body || {};
340
+ let tmpRecord =
341
+ {
342
+ IDSource: tmpIDSource,
343
+ Name: tmpBody.Name || 'Untitled',
344
+ DocumentType: tmpBody.DocumentType || 'markdown',
345
+ MimeType: tmpBody.MimeType || 'text/markdown',
346
+ StorageKey: '',
347
+ Description: tmpBody.Description || '',
348
+ Content: tmpBody.Content || ''
349
+ };
350
+
351
+ let tmpQuery = this.fable.DAL.SourceDocumentation.query.clone()
352
+ .addRecord(tmpRecord);
353
+
354
+ this.fable.DAL.SourceDocumentation.doCreate(tmpQuery,
355
+ (pError, pQuery, pQueryRead, pCreatedRecord) =>
356
+ {
357
+ if (pError)
358
+ {
359
+ this.fable.log.error(`SourceManager error creating documentation for source ${tmpIDSource}: ${pError}`);
360
+ pResponse.send({ Error: pError.message || pError });
361
+ return fNext();
362
+ }
363
+ pResponse.send({ Success: true, Documentation: pCreatedRecord });
364
+ return fNext();
365
+ });
366
+ });
367
+
368
+ // GET /facto/source/:IDSource/documentation/:IDSourceDocumentation -- read a single doc
369
+ pOratorServiceServer.doGet(`${tmpRoutePrefix}/source/:IDSource/documentation/:IDSourceDocumentation`,
370
+ (pRequest, pResponse, fNext) =>
371
+ {
372
+ let tmpIDSource = parseInt(pRequest.params.IDSource, 10);
373
+ let tmpIDDoc = parseInt(pRequest.params.IDSourceDocumentation, 10);
374
+ if (isNaN(tmpIDSource) || tmpIDSource < 1 || isNaN(tmpIDDoc) || tmpIDDoc < 1)
375
+ {
376
+ pResponse.send({ Error: 'Invalid IDSource or IDSourceDocumentation parameter' });
377
+ return fNext();
378
+ }
379
+
380
+ if (!this.fable.DAL || !this.fable.DAL.SourceDocumentation)
381
+ {
382
+ pResponse.send({ Error: 'SourceDocumentation DAL not initialized' });
383
+ return fNext();
384
+ }
385
+
386
+ let tmpQuery = this.fable.DAL.SourceDocumentation.query.clone()
387
+ .addFilter('IDSourceDocumentation', tmpIDDoc)
388
+ .addFilter('IDSource', tmpIDSource)
389
+ .addFilter('Deleted', 0);
390
+
391
+ this.fable.DAL.SourceDocumentation.doRead(tmpQuery,
392
+ (pError, pQuery, pRecord) =>
393
+ {
394
+ if (pError)
395
+ {
396
+ pResponse.send({ Error: pError.message || pError });
397
+ return fNext();
398
+ }
399
+ if (!pRecord)
400
+ {
401
+ pResponse.send({ Error: `Documentation ${tmpIDDoc} not found for source ${tmpIDSource}` });
402
+ return fNext();
403
+ }
404
+ pResponse.send({ Documentation: pRecord });
405
+ return fNext();
406
+ });
407
+ });
408
+
409
+ // PUT /facto/source/:IDSource/documentation/:IDSourceDocumentation -- update a doc
410
+ pOratorServiceServer.doPut(`${tmpRoutePrefix}/source/:IDSource/documentation/:IDSourceDocumentation`,
411
+ (pRequest, pResponse, fNext) =>
412
+ {
413
+ let tmpIDSource = parseInt(pRequest.params.IDSource, 10);
414
+ let tmpIDDoc = parseInt(pRequest.params.IDSourceDocumentation, 10);
415
+ if (isNaN(tmpIDSource) || tmpIDSource < 1 || isNaN(tmpIDDoc) || tmpIDDoc < 1)
416
+ {
417
+ pResponse.send({ Error: 'Invalid IDSource or IDSourceDocumentation parameter' });
418
+ return fNext();
419
+ }
420
+
421
+ if (!this.fable.DAL || !this.fable.DAL.SourceDocumentation)
422
+ {
423
+ pResponse.send({ Error: 'SourceDocumentation DAL not initialized' });
424
+ return fNext();
425
+ }
426
+
427
+ let tmpBody = pRequest.body || {};
428
+ let tmpUpdateRecord = { IDSourceDocumentation: tmpIDDoc };
429
+ if (tmpBody.hasOwnProperty('Name')) tmpUpdateRecord.Name = tmpBody.Name;
430
+ if (tmpBody.hasOwnProperty('Content')) tmpUpdateRecord.Content = tmpBody.Content;
431
+ if (tmpBody.hasOwnProperty('Description')) tmpUpdateRecord.Description = tmpBody.Description;
432
+ if (tmpBody.hasOwnProperty('DocumentType')) tmpUpdateRecord.DocumentType = tmpBody.DocumentType;
433
+
434
+ let tmpQuery = this.fable.DAL.SourceDocumentation.query.clone()
435
+ .addRecord(tmpUpdateRecord);
436
+
437
+ this.fable.DAL.SourceDocumentation.doUpdate(tmpQuery,
438
+ (pError, pQuery, pQueryRead, pUpdatedRecord) =>
439
+ {
440
+ if (pError)
441
+ {
442
+ this.fable.log.error(`SourceManager error updating documentation ${tmpIDDoc}: ${pError}`);
443
+ pResponse.send({ Error: pError.message || pError });
444
+ return fNext();
445
+ }
446
+ pResponse.send({ Success: true, Documentation: pUpdatedRecord });
447
+ return fNext();
448
+ });
449
+ });
450
+
451
+ // DELETE /facto/source/:IDSource/documentation/:IDSourceDocumentation -- soft-delete a doc
452
+ pOratorServiceServer.doDel(`${tmpRoutePrefix}/source/:IDSource/documentation/:IDSourceDocumentation`,
453
+ (pRequest, pResponse, fNext) =>
454
+ {
455
+ let tmpIDSource = parseInt(pRequest.params.IDSource, 10);
456
+ let tmpIDDoc = parseInt(pRequest.params.IDSourceDocumentation, 10);
457
+ if (isNaN(tmpIDSource) || tmpIDSource < 1 || isNaN(tmpIDDoc) || tmpIDDoc < 1)
458
+ {
459
+ pResponse.send({ Error: 'Invalid IDSource or IDSourceDocumentation parameter' });
460
+ return fNext();
461
+ }
462
+
463
+ if (!this.fable.DAL || !this.fable.DAL.SourceDocumentation)
464
+ {
465
+ pResponse.send({ Error: 'SourceDocumentation DAL not initialized' });
466
+ return fNext();
467
+ }
468
+
469
+ let tmpQuery = this.fable.DAL.SourceDocumentation.query.clone()
470
+ .addRecord({ IDSourceDocumentation: tmpIDDoc, Deleted: 1, DeleteDate: new Date().toISOString() });
471
+
472
+ this.fable.DAL.SourceDocumentation.doUpdate(tmpQuery,
473
+ (pError, pQuery, pQueryRead, pRecord) =>
474
+ {
475
+ if (pError)
476
+ {
477
+ this.fable.log.error(`SourceManager error deleting documentation ${tmpIDDoc}: ${pError}`);
478
+ pResponse.send({ Error: pError.message || pError });
479
+ return fNext();
480
+ }
481
+ pResponse.send({ Success: true, Deleted: tmpIDDoc });
482
+ return fNext();
483
+ });
484
+ });
485
+
486
+ // GET /facto/source/:IDSource/catalog-context -- reverse-lookup catalog entries linked to this source
487
+ pOratorServiceServer.doGet(`${tmpRoutePrefix}/source/:IDSource/catalog-context`,
488
+ (pRequest, pResponse, fNext) =>
489
+ {
490
+ let tmpIDSource = parseInt(pRequest.params.IDSource, 10);
491
+ if (isNaN(tmpIDSource) || tmpIDSource < 1)
492
+ {
493
+ pResponse.send({ Error: 'Invalid IDSource parameter', CatalogEntries: [], DatasetDefinitions: [] });
494
+ return fNext();
495
+ }
496
+
497
+ if (!this.fable.DAL || !this.fable.DAL.CatalogDatasetDefinition)
498
+ {
499
+ pResponse.send({ Error: 'CatalogDatasetDefinition DAL not initialized', CatalogEntries: [], DatasetDefinitions: [] });
500
+ return fNext();
501
+ }
502
+
503
+ let tmpAnticipate = this.fable.newAnticipate();
504
+ let tmpResult = { IDSource: tmpIDSource, CatalogEntries: [], DatasetDefinitions: [] };
505
+
506
+ // Find all CatalogDatasetDefinition records linked to this source
507
+ tmpAnticipate.anticipate(
508
+ (fStep) =>
509
+ {
510
+ let tmpQuery = this.fable.DAL.CatalogDatasetDefinition.query.clone()
511
+ .addFilter('IDSource', tmpIDSource)
512
+ .addFilter('Deleted', 0);
513
+
514
+ this.fable.DAL.CatalogDatasetDefinition.doReads(tmpQuery,
515
+ (pError, pQuery, pRecords) =>
516
+ {
517
+ if (!pError && pRecords)
518
+ {
519
+ tmpResult.DatasetDefinitions = pRecords;
520
+ }
521
+ return fStep();
522
+ });
523
+ });
524
+
525
+ tmpAnticipate.wait(
526
+ (pError) =>
527
+ {
528
+ if (pError || tmpResult.DatasetDefinitions.length === 0)
529
+ {
530
+ pResponse.send(tmpResult);
531
+ return fNext();
532
+ }
533
+
534
+ // Collect unique catalog entry IDs
535
+ let tmpEntryIDs = {};
536
+ for (let i = 0; i < tmpResult.DatasetDefinitions.length; i++)
537
+ {
538
+ let tmpEntryID = tmpResult.DatasetDefinitions[i].IDSourceCatalogEntry;
539
+ if (tmpEntryID && tmpEntryID > 0)
540
+ {
541
+ tmpEntryIDs[tmpEntryID] = true;
542
+ }
543
+ }
544
+
545
+ let tmpEntryIDList = Object.keys(tmpEntryIDs);
546
+ if (tmpEntryIDList.length === 0)
547
+ {
548
+ pResponse.send(tmpResult);
549
+ return fNext();
550
+ }
551
+
552
+ // Load each catalog entry
553
+ let tmpEntryAnticipate = this.fable.newAnticipate();
554
+ for (let i = 0; i < tmpEntryIDList.length; i++)
555
+ {
556
+ let tmpEntryID = parseInt(tmpEntryIDList[i], 10);
557
+ tmpEntryAnticipate.anticipate(
558
+ (fEntryStep) =>
559
+ {
560
+ let tmpEntryQuery = this.fable.DAL.SourceCatalogEntry.query.clone()
561
+ .addFilter('IDSourceCatalogEntry', tmpEntryID);
562
+
563
+ this.fable.DAL.SourceCatalogEntry.doRead(tmpEntryQuery,
564
+ (pEntryError, pEntryQuery, pEntry) =>
565
+ {
566
+ if (!pEntryError && pEntry)
567
+ {
568
+ tmpResult.CatalogEntries.push(pEntry);
569
+ }
570
+ return fEntryStep();
571
+ });
572
+ });
573
+ }
574
+
575
+ tmpEntryAnticipate.wait(
576
+ (pEntryError) =>
577
+ {
578
+ pResponse.send(tmpResult);
579
+ return fNext();
580
+ });
581
+ });
582
+ });
583
+
584
+ // GET /facto/catalog/source-links -- returns a map of IDSourceCatalogEntry -> IDSource for all provisioned entries
585
+ pOratorServiceServer.doGet(`${tmpRoutePrefix}/catalog/source-links`,
586
+ (pRequest, pResponse, fNext) =>
587
+ {
588
+ if (!this.fable.DAL || !this.fable.DAL.CatalogDatasetDefinition)
589
+ {
590
+ pResponse.send({ Links: {} });
591
+ return fNext();
592
+ }
593
+
594
+ let tmpQuery = this.fable.DAL.CatalogDatasetDefinition.query.clone()
595
+ .addFilter('Deleted', 0);
596
+
597
+ this.fable.DAL.CatalogDatasetDefinition.doReads(tmpQuery,
598
+ (pError, pQuery, pRecords) =>
599
+ {
600
+ let tmpLinks = {};
601
+ if (!pError && pRecords)
602
+ {
603
+ for (let i = 0; i < pRecords.length; i++)
604
+ {
605
+ let tmpRec = pRecords[i];
606
+ if (tmpRec.IDSource && tmpRec.IDSource > 0 && tmpRec.IDSourceCatalogEntry)
607
+ {
608
+ tmpLinks[tmpRec.IDSourceCatalogEntry] = tmpRec.IDSource;
609
+ }
610
+ }
611
+ }
612
+ pResponse.send({ Links: tmpLinks });
613
+ return fNext();
614
+ });
615
+ });
616
+
617
+ // POST /facto/source/:IDSource/documentation/upload -- upload a file (image, pdf, etc.)
618
+ // Body: { Filename, ContentType, Data } where Data is base64-encoded file content
619
+ pOratorServiceServer.doPost(`${tmpRoutePrefix}/source/:IDSource/documentation/upload`,
620
+ (pRequest, pResponse, fNext) =>
621
+ {
622
+ let tmpIDSource = parseInt(pRequest.params.IDSource, 10);
623
+ if (isNaN(tmpIDSource) || tmpIDSource < 1)
624
+ {
625
+ pResponse.send({ Error: 'Invalid IDSource parameter' });
626
+ return fNext();
627
+ }
628
+
629
+ let tmpFilename = pRequest.body && pRequest.body.Filename;
630
+ let tmpData = pRequest.body && pRequest.body.Data;
631
+ if (!tmpFilename || !tmpData)
632
+ {
633
+ pResponse.send({ Error: 'Filename and Data (base64) are required' });
634
+ return fNext();
635
+ }
636
+
637
+ let tmpPath = require('path');
638
+ let tmpFS = require('fs');
639
+
640
+ // Sanitize filename: keep only alphanumeric, hyphens, underscores, dots
641
+ let tmpSafeFilename = tmpFilename.replace(/[^a-zA-Z0-9._-]/g, '_');
642
+ // Prefix with timestamp to avoid collisions
643
+ let tmpTimestamp = Date.now();
644
+ let tmpFinalFilename = tmpTimestamp + '-' + tmpSafeFilename;
645
+
646
+ // Build target directory: data/source-docs/{IDSource}/
647
+ let tmpDataDir = this.fable.settings.DataDirectory || tmpPath.join(process.cwd(), 'data');
648
+ let tmpSourceDocsDir = tmpPath.join(tmpDataDir, 'source-docs', String(tmpIDSource));
649
+
650
+ // Ensure directory exists
651
+ tmpFS.mkdirSync(tmpSourceDocsDir, { recursive: true });
652
+
653
+ // Decode base64 and write
654
+ let tmpBuffer = Buffer.from(tmpData, 'base64');
655
+ let tmpFilePath = tmpPath.join(tmpSourceDocsDir, tmpFinalFilename);
656
+
657
+ tmpFS.writeFile(tmpFilePath, tmpBuffer,
658
+ (pError) =>
659
+ {
660
+ if (pError)
661
+ {
662
+ this.fable.log.error(`SourceManager error writing file ${tmpFilePath}: ${pError}`);
663
+ pResponse.send({ Error: 'Failed to write file: ' + pError.message });
664
+ return fNext();
665
+ }
666
+
667
+ let tmpURL = `${tmpRoutePrefix}/source/${tmpIDSource}/documentation/files/${tmpFinalFilename}`;
668
+ pResponse.send({ Success: true, URL: tmpURL, Filename: tmpFinalFilename });
669
+ return fNext();
670
+ });
671
+ });
672
+
673
+ // GET /facto/source/:IDSource/documentation/files/:Filename -- serve an uploaded file
674
+ pOratorServiceServer.doGet(`${tmpRoutePrefix}/source/:IDSource/documentation/files/:Filename`,
675
+ (pRequest, pResponse, fNext) =>
676
+ {
677
+ let tmpIDSource = parseInt(pRequest.params.IDSource, 10);
678
+ let tmpFilename = pRequest.params.Filename;
679
+ if (isNaN(tmpIDSource) || tmpIDSource < 1 || !tmpFilename)
680
+ {
681
+ pResponse.send(404, { Error: 'Not found' });
682
+ return fNext();
683
+ }
684
+
685
+ let tmpPath = require('path');
686
+ let tmpFS = require('fs');
687
+
688
+ // Sanitize to prevent path traversal
689
+ let tmpSafeFilename = tmpPath.basename(tmpFilename);
690
+ let tmpDataDir = this.fable.settings.DataDirectory || tmpPath.join(process.cwd(), 'data');
691
+ let tmpFilePath = tmpPath.join(tmpDataDir, 'source-docs', String(tmpIDSource), tmpSafeFilename);
692
+
693
+ if (!tmpFS.existsSync(tmpFilePath))
694
+ {
695
+ pResponse.send(404, { Error: 'File not found' });
696
+ return fNext();
697
+ }
698
+
699
+ // Determine content type from extension
700
+ let tmpExt = tmpPath.extname(tmpSafeFilename).toLowerCase();
701
+ let tmpContentTypes =
702
+ {
703
+ '.png': 'image/png',
704
+ '.jpg': 'image/jpeg',
705
+ '.jpeg': 'image/jpeg',
706
+ '.gif': 'image/gif',
707
+ '.webp': 'image/webp',
708
+ '.svg': 'image/svg+xml',
709
+ '.pdf': 'application/pdf',
710
+ '.md': 'text/markdown',
711
+ '.txt': 'text/plain'
712
+ };
713
+ let tmpContentType = tmpContentTypes[tmpExt] || 'application/octet-stream';
714
+
715
+ let tmpFileData = tmpFS.readFileSync(tmpFilePath);
716
+ pResponse.setHeader('Content-Type', tmpContentType);
717
+ pResponse.setHeader('Content-Length', tmpFileData.length);
718
+ pResponse.writeHead(200);
719
+ pResponse.write(tmpFileData);
720
+ pResponse.end();
721
+ return fNext();
722
+ });
723
+
724
+ this.fable.log.info(`SourceManager routes connected at ${tmpRoutePrefix}/source(s)/*`);
725
+ }
726
+ }
727
+
728
+ module.exports = RetoldFactoSourceManager;
729
+ module.exports.serviceType = 'RetoldFactoSourceManager';
730
+ module.exports.default_configuration = defaultSourceManagerOptions;