retold-facto 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/.claude/launch.json +11 -0
  2. package/.dockerignore +8 -0
  3. package/.quackage.json +19 -0
  4. package/Dockerfile +26 -0
  5. package/bin/retold-facto.js +909 -0
  6. package/examples/facto-government-data.sqlite +0 -0
  7. package/examples/government-data-catalog.json +137 -0
  8. package/examples/government-data-loader.js +1432 -0
  9. package/package.json +91 -0
  10. package/scripts/facto-download.js +425 -0
  11. package/source/Retold-Facto.js +1042 -0
  12. package/source/services/Retold-Facto-BeaconProvider.js +511 -0
  13. package/source/services/Retold-Facto-CatalogManager.js +1252 -0
  14. package/source/services/Retold-Facto-DataLakeService.js +1642 -0
  15. package/source/services/Retold-Facto-DatasetManager.js +417 -0
  16. package/source/services/Retold-Facto-IngestEngine.js +1315 -0
  17. package/source/services/Retold-Facto-ProjectionEngine.js +3960 -0
  18. package/source/services/Retold-Facto-RecordManager.js +360 -0
  19. package/source/services/Retold-Facto-SchemaManager.js +1110 -0
  20. package/source/services/Retold-Facto-SourceFolderScanner.js +2243 -0
  21. package/source/services/Retold-Facto-SourceManager.js +730 -0
  22. package/source/services/Retold-Facto-StoreConnectionManager.js +441 -0
  23. package/source/services/Retold-Facto-ThroughputMonitor.js +478 -0
  24. package/source/services/web-app/codemirror-entry.js +7 -0
  25. package/source/services/web-app/pict-app/Pict-Application-Facto-Configuration.json +9 -0
  26. package/source/services/web-app/pict-app/Pict-Application-Facto.js +70 -0
  27. package/source/services/web-app/pict-app/Pict-Facto-Bundle.js +11 -0
  28. package/source/services/web-app/pict-app/providers/Pict-Provider-Facto-UI.js +66 -0
  29. package/source/services/web-app/pict-app/providers/Pict-Provider-Facto.js +69 -0
  30. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Catalog.js +93 -0
  31. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Connections.js +42 -0
  32. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Datasets.js +605 -0
  33. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Projections.js +188 -0
  34. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Scanner.js +80 -0
  35. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Schema.js +116 -0
  36. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Sources.js +104 -0
  37. package/source/services/web-app/pict-app/views/PictView-Facto-Catalog.js +526 -0
  38. package/source/services/web-app/pict-app/views/PictView-Facto-Datasets.js +173 -0
  39. package/source/services/web-app/pict-app/views/PictView-Facto-Ingest.js +259 -0
  40. package/source/services/web-app/pict-app/views/PictView-Facto-Layout.js +191 -0
  41. package/source/services/web-app/pict-app/views/PictView-Facto-Projections.js +231 -0
  42. package/source/services/web-app/pict-app/views/PictView-Facto-Records.js +326 -0
  43. package/source/services/web-app/pict-app/views/PictView-Facto-Scanner.js +624 -0
  44. package/source/services/web-app/pict-app/views/PictView-Facto-Sources.js +201 -0
  45. package/source/services/web-app/pict-app/views/PictView-Facto-Throughput.js +456 -0
  46. package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full-Configuration.json +14 -0
  47. package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full.js +391 -0
  48. package/source/services/web-app/pict-app-full/providers/PictRouter-Facto-Configuration.json +56 -0
  49. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-BottomBar.js +68 -0
  50. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Connections.js +340 -0
  51. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboard.js +149 -0
  52. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboards.js +819 -0
  53. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Datasets.js +178 -0
  54. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-IngestJobs.js +99 -0
  55. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Layout.js +62 -0
  56. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-MappingEditor.js +158 -0
  57. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-ProjectionDetail.js +1120 -0
  58. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Projections.js +172 -0
  59. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-QueryPanel.js +119 -0
  60. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-RecordViewer.js +663 -0
  61. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Records.js +648 -0
  62. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Scanner.js +1017 -0
  63. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDetail.js +1404 -0
  64. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDocEditor.js +1036 -0
  65. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaEditor.js +636 -0
  66. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaResearch.js +357 -0
  67. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceDetail.js +822 -0
  68. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceEditor.js +1036 -0
  69. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceResearch.js +487 -0
  70. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Sources.js +165 -0
  71. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Throughput.js +439 -0
  72. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-TopBar.js +335 -0
  73. package/source/services/web-app/pict-app-full/views/projections/Facto-Projections-Constants.js +71 -0
  74. package/source/services/web-app/web/chart.min.js +20 -0
  75. package/source/services/web-app/web/codemirror-bundle.js +30099 -0
  76. package/source/services/web-app/web/css/facto-themes.css +467 -0
  77. package/source/services/web-app/web/css/facto.css +502 -0
  78. package/source/services/web-app/web/index.html +28 -0
  79. package/source/services/web-app/web/retold-facto.js +12138 -0
  80. package/source/services/web-app/web/retold-facto.js.map +1 -0
  81. package/source/services/web-app/web/retold-facto.min.js +2 -0
  82. package/source/services/web-app/web/retold-facto.min.js.map +1 -0
  83. package/source/services/web-app/web/simple/index.html +17 -0
  84. package/test/Facto_Browser_Integration_tests.js +798 -0
  85. package/test/RetoldFacto_tests.js +4117 -0
  86. package/test/fixtures/weather-readings.csv +17 -0
  87. package/test/fixtures/weather-stations.csv +9 -0
  88. package/test/model/MeadowModel-Extended.json +8497 -0
  89. package/test/model/MeadowModel-PICT.json +1 -0
  90. package/test/model/MeadowModel.json +1355 -0
  91. package/test/model/ddl/Facto.ddl +225 -0
  92. package/test/model/fable-configuration.json +14 -0
@@ -0,0 +1,624 @@
1
+ const libPictView = require('pict-view');
2
+
3
+ class FactoScannerView extends libPictView
4
+ {
5
+ constructor(pFable, pOptions, pServiceHash)
6
+ {
7
+ super(pFable, pOptions, pServiceHash);
8
+ }
9
+
10
+ onAfterRender()
11
+ {
12
+ this.loadScannerState();
13
+ }
14
+
15
+ loadScannerState()
16
+ {
17
+ // Load scan paths and discovered datasets
18
+ Promise.all(
19
+ [
20
+ this.pict.providers.Facto.loadScannerPaths(),
21
+ this.pict.providers.Facto.loadScannerDatasets()
22
+ ]).then(
23
+ () =>
24
+ {
25
+ this.refreshPathsList();
26
+ this.refreshDatasetsList();
27
+ }).catch(
28
+ (pError) =>
29
+ {
30
+ this.pict.views['Pict-Section-Modal'].toast('Error loading scanner state: ' + pError.message, {type: 'error'});
31
+ });
32
+ }
33
+
34
+ refreshPathsList()
35
+ {
36
+ let tmpContainer = document.getElementById('facto-scanner-paths-list');
37
+ if (!tmpContainer) return;
38
+
39
+ let tmpPaths = this.pict.AppData.Facto.ScannerPaths || [];
40
+
41
+ if (tmpPaths.length === 0)
42
+ {
43
+ tmpContainer.innerHTML = '<p style="color:#888; font-style:italic;">No scan paths configured. Add a folder path to discover datasets.</p>';
44
+ return;
45
+ }
46
+
47
+ let tmpHtml = '<table><thead><tr><th>Path</th><th>Datasets</th><th>Last Scanned</th><th>Actions</th></tr></thead><tbody>';
48
+ for (let i = 0; i < tmpPaths.length; i++)
49
+ {
50
+ let tmpPath = tmpPaths[i];
51
+ let tmpLastScanned = tmpPath.LastScannedAt ? new Date(tmpPath.LastScannedAt).toLocaleString() : 'Never';
52
+ tmpHtml += '<tr>';
53
+ tmpHtml += '<td style="font-family:monospace; font-size:0.85em;">' + this.escapeHtml(tmpPath.Path) + '</td>';
54
+ tmpHtml += '<td>' + (tmpPath.DatasetCount || 0) + '</td>';
55
+ tmpHtml += '<td>' + tmpLastScanned + '</td>';
56
+ tmpHtml += '<td>';
57
+ tmpHtml += '<button class="danger" style="padding:4px 8px; font-size:0.8em;" onclick="pict.views[\'Facto-Scanner\'].removePath(\'' + this.escapeAttr(tmpPath.Path) + '\')">Remove</button>';
58
+ tmpHtml += '</td>';
59
+ tmpHtml += '</tr>';
60
+ }
61
+ tmpHtml += '</tbody></table>';
62
+ tmpContainer.innerHTML = tmpHtml;
63
+ }
64
+
65
+ refreshDatasetsList()
66
+ {
67
+ let tmpContainer = document.getElementById('facto-scanner-datasets-list');
68
+ if (!tmpContainer) return;
69
+
70
+ let tmpDatasets = this.pict.AppData.Facto.ScannerDatasets || [];
71
+
72
+ if (tmpDatasets.length === 0)
73
+ {
74
+ tmpContainer.innerHTML = '<p style="color:#888; font-style:italic;">No datasets discovered yet. Add a scan path containing dataset folders with README.md files.</p>';
75
+ this.updateSummary(0, 0, 0, 0);
76
+ return;
77
+ }
78
+
79
+ // Compute summary stats
80
+ let tmpDiscovered = 0;
81
+ let tmpProvisioned = 0;
82
+ let tmpIngested = 0;
83
+ let tmpWithData = 0;
84
+ for (let i = 0; i < tmpDatasets.length; i++)
85
+ {
86
+ if (tmpDatasets[i].Status === 'Discovered') tmpDiscovered++;
87
+ if (tmpDatasets[i].Status === 'Provisioned') tmpProvisioned++;
88
+ if (tmpDatasets[i].Status === 'Ingested') tmpIngested++;
89
+ if (tmpDatasets[i].HasData) tmpWithData++;
90
+ }
91
+ this.updateSummary(tmpDatasets.length, tmpDiscovered, tmpProvisioned, tmpWithData);
92
+
93
+ // Apply search filter
94
+ let tmpSearchEl = document.getElementById('facto-scanner-search');
95
+ let tmpSearchTerm = tmpSearchEl ? tmpSearchEl.value.toLowerCase() : '';
96
+
97
+ let tmpStatusFilter = document.getElementById('facto-scanner-status-filter');
98
+ let tmpStatusValue = tmpStatusFilter ? tmpStatusFilter.value : '';
99
+
100
+ let tmpFiltered = tmpDatasets;
101
+ if (tmpSearchTerm)
102
+ {
103
+ tmpFiltered = tmpFiltered.filter(
104
+ (pDS) =>
105
+ {
106
+ let tmpText = ((pDS.FolderName || '') + ' ' + (pDS.Title || '') + ' ' + (pDS.Provider || '')).toLowerCase();
107
+ return tmpText.indexOf(tmpSearchTerm) > -1;
108
+ });
109
+ }
110
+ if (tmpStatusValue)
111
+ {
112
+ tmpFiltered = tmpFiltered.filter(
113
+ (pDS) =>
114
+ {
115
+ return pDS.Status === tmpStatusValue;
116
+ });
117
+ }
118
+
119
+ let tmpHtml = '<table><thead><tr>';
120
+ tmpHtml += '<th><input type="checkbox" id="facto-scanner-select-all" onclick="pict.views[\'Facto-Scanner\'].toggleSelectAll(this.checked)"></th>';
121
+ tmpHtml += '<th>Name</th><th>Title</th><th>Provider</th><th>Format</th><th>Data</th><th>Status</th><th>Actions</th>';
122
+ tmpHtml += '</tr></thead><tbody>';
123
+
124
+ for (let i = 0; i < tmpFiltered.length; i++)
125
+ {
126
+ let tmpDS = tmpFiltered[i];
127
+ let tmpStatusClass = this.getStatusBadgeClass(tmpDS.Status);
128
+ let tmpDataInfo = tmpDS.HasData
129
+ ? (tmpDS.DataFileCount + ' file' + (tmpDS.DataFileCount !== 1 ? 's' : '') + ' (' + (tmpDS.TotalDataSizeFormatted || '0 B') + ')')
130
+ : '<span style="color:#dc3545;">No data</span>';
131
+ let tmpFormat = tmpDS.DataFormat ? (tmpDS.DataFormat.Format || 'unknown') : 'unknown';
132
+
133
+ tmpHtml += '<tr>';
134
+ tmpHtml += '<td><input type="checkbox" class="facto-scanner-checkbox" data-folder="' + this.escapeAttr(tmpDS.FolderName) + '"></td>';
135
+ tmpHtml += '<td style="font-family:monospace; font-size:0.85em;">' + this.escapeHtml(tmpDS.FolderName) + '</td>';
136
+ tmpHtml += '<td>' + this.escapeHtml((tmpDS.Title || '').substring(0, 50)) + '</td>';
137
+ tmpHtml += '<td>' + this.escapeHtml((tmpDS.Provider || '').substring(0, 30)) + '</td>';
138
+ tmpHtml += '<td>' + tmpFormat + '</td>';
139
+ tmpHtml += '<td>' + tmpDataInfo + '</td>';
140
+ tmpHtml += '<td><span class="badge ' + tmpStatusClass + '">' + tmpDS.Status + '</span></td>';
141
+ tmpHtml += '<td>';
142
+ let tmpEscFolder = this.escapeAttr(tmpDS.FolderName);
143
+ tmpHtml += '<div class="facto-row-actions" id="facto-row-actions-' + tmpEscFolder + '">';
144
+ tmpHtml += '<button class="facto-row-actions-trigger" onclick="pict.views[\'Facto-Scanner\'].toggleRowMenu(event, \'' + tmpEscFolder + '\')" title="Actions">&#8942;</button>';
145
+ tmpHtml += '<div class="facto-row-actions-menu">';
146
+ tmpHtml += '<button onclick="pict.views[\'Facto-Scanner\'].viewDetail(\'' + tmpEscFolder + '\'); pict.views[\'Facto-Scanner\'].closeRowMenus();">Detail</button>';
147
+ if (tmpDS.Status === 'Discovered')
148
+ {
149
+ tmpHtml += '<button class="facto-action-success" onclick="pict.views[\'Facto-Scanner\'].provisionOne(\'' + tmpEscFolder + '\'); pict.views[\'Facto-Scanner\'].closeRowMenus();">Provision</button>';
150
+ }
151
+ tmpHtml += '</div>';
152
+ tmpHtml += '</div>';
153
+ tmpHtml += '</td>';
154
+ tmpHtml += '</tr>';
155
+ }
156
+ tmpHtml += '</tbody></table>';
157
+ tmpHtml += '<p style="color:#888; font-size:0.85em; margin-top:8px;">Showing ' + tmpFiltered.length + ' of ' + tmpDatasets.length + ' dataset(s)</p>';
158
+
159
+ tmpContainer.innerHTML = tmpHtml;
160
+ }
161
+
162
+ updateSummary(pTotal, pDiscovered, pProvisioned, pWithData)
163
+ {
164
+ let tmpEl = document.getElementById('facto-scanner-summary');
165
+ if (!tmpEl) return;
166
+ tmpEl.innerHTML = '<strong>' + pTotal + '</strong> discovered &nbsp;|&nbsp; <strong>' + pProvisioned + '</strong> provisioned &nbsp;|&nbsp; <strong>' + pWithData + '</strong> with data';
167
+ }
168
+
169
+ getStatusBadgeClass(pStatus)
170
+ {
171
+ if (pStatus === 'Discovered') return 'badge-raw';
172
+ if (pStatus === 'Provisioned') return 'badge-compositional';
173
+ if (pStatus === 'Ingested') return 'badge-projection';
174
+ if (pStatus === 'Error') return 'badge-derived';
175
+ return '';
176
+ }
177
+
178
+ escapeHtml(pStr)
179
+ {
180
+ if (!pStr) return '';
181
+ return pStr.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
182
+ }
183
+
184
+ escapeAttr(pStr)
185
+ {
186
+ if (!pStr) return '';
187
+ return pStr.replace(/\\/g, '\\\\').replace(/'/g, "\\'").replace(/"/g, '&quot;');
188
+ }
189
+
190
+ // ================================================================
191
+ // Actions
192
+ // ================================================================
193
+
194
+ addPath()
195
+ {
196
+ let tmpPathInput = document.getElementById('facto-scanner-path-input');
197
+ let tmpPath = tmpPathInput ? tmpPathInput.value.trim() : '';
198
+
199
+ if (!tmpPath)
200
+ {
201
+ this.pict.views['Pict-Section-Modal'].toast('Enter a folder path to scan', {type: 'warning'});
202
+ return;
203
+ }
204
+
205
+ this.pict.views['Pict-Section-Modal'].toast('Scanning ' + tmpPath + '...', {type: 'info'});
206
+
207
+ this.pict.providers.Facto.addScannerPath(tmpPath).then(
208
+ (pResponse) =>
209
+ {
210
+ if (pResponse && pResponse.Error)
211
+ {
212
+ this.pict.views['Pict-Section-Modal'].toast('Error: ' + pResponse.Error, {type: 'error'});
213
+ return;
214
+ }
215
+ let tmpResult = pResponse.ScanResult || {};
216
+ this.pict.views['Pict-Section-Modal'].toast('Scanned! Found ' + (tmpResult.DatasetsFound || 0) + ' dataset(s) in ' + (tmpResult.FoldersScanned || 0) + ' folder(s)', {type: 'success'});
217
+ if (tmpPathInput) tmpPathInput.value = '';
218
+ this.loadScannerState();
219
+ }).catch(
220
+ (pError) =>
221
+ {
222
+ this.pict.views['Pict-Section-Modal'].toast('Error: ' + pError.message, {type: 'error'});
223
+ });
224
+ }
225
+
226
+ async removePath(pPath)
227
+ {
228
+ let tmpConfirmed = await this.pict.views['Pict-Section-Modal'].confirm('Remove scan path and its discovered datasets?\n\n' + pPath, { title: 'Remove Path', confirmLabel: 'Remove', dangerous: true });
229
+ if (!tmpConfirmed) return;
230
+
231
+ this.pict.providers.Facto.removeScannerPath(pPath).then(
232
+ (pResponse) =>
233
+ {
234
+ if (pResponse && pResponse.Error)
235
+ {
236
+ this.pict.views['Pict-Section-Modal'].toast('Error: ' + pResponse.Error, {type: 'error'});
237
+ return;
238
+ }
239
+ this.pict.views['Pict-Section-Modal'].toast('Path removed', {type: 'success'});
240
+ this.loadScannerState();
241
+ }).catch(
242
+ (pError) =>
243
+ {
244
+ this.pict.views['Pict-Section-Modal'].toast('Error: ' + pError.message, {type: 'error'});
245
+ });
246
+ }
247
+
248
+ rescanAll()
249
+ {
250
+ this.pict.views['Pict-Section-Modal'].toast('Re-scanning all paths...', {type: 'info'});
251
+
252
+ this.pict.providers.Facto.rescanPaths().then(
253
+ (pResponse) =>
254
+ {
255
+ if (pResponse && pResponse.Error)
256
+ {
257
+ this.pict.views['Pict-Section-Modal'].toast('Error: ' + pResponse.Error, {type: 'error'});
258
+ return;
259
+ }
260
+ this.pict.views['Pict-Section-Modal'].toast('Re-scan complete', {type: 'success'});
261
+ this.loadScannerState();
262
+ }).catch(
263
+ (pError) =>
264
+ {
265
+ this.pict.views['Pict-Section-Modal'].toast('Error: ' + pError.message, {type: 'error'});
266
+ });
267
+ }
268
+
269
+ filterDatasets()
270
+ {
271
+ this.refreshDatasetsList();
272
+ }
273
+
274
+ toggleSelectAll(pChecked)
275
+ {
276
+ let tmpCheckboxes = document.querySelectorAll('.facto-scanner-checkbox');
277
+ for (let i = 0; i < tmpCheckboxes.length; i++)
278
+ {
279
+ tmpCheckboxes[i].checked = pChecked;
280
+ }
281
+ }
282
+
283
+ getSelectedFolderNames()
284
+ {
285
+ let tmpCheckboxes = document.querySelectorAll('.facto-scanner-checkbox:checked');
286
+ let tmpNames = [];
287
+ for (let i = 0; i < tmpCheckboxes.length; i++)
288
+ {
289
+ tmpNames.push(tmpCheckboxes[i].getAttribute('data-folder'));
290
+ }
291
+ return tmpNames;
292
+ }
293
+
294
+ provisionOne(pFolderName)
295
+ {
296
+ this.pict.views['Pict-Section-Modal'].toast('Provisioning ' + pFolderName + '...', {type: 'info'});
297
+
298
+ this.pict.providers.Facto.provisionScannerDataset(pFolderName).then(
299
+ (pResponse) =>
300
+ {
301
+ if (pResponse && pResponse.Error)
302
+ {
303
+ this.pict.views['Pict-Section-Modal'].toast('Error: ' + pResponse.Error, {type: 'error'});
304
+ return;
305
+ }
306
+ this.pict.views['Pict-Section-Modal'].toast('Provisioned ' + pFolderName + ' (Source #' + (pResponse.Source ? pResponse.Source.IDSource : '?') + ', Dataset #' + (pResponse.Dataset ? pResponse.Dataset.IDDataset : '?') + ')', {type: 'success'});
307
+ this.loadScannerState();
308
+ // Refresh sources/datasets views
309
+ this.pict.providers.FactoUI.refreshDataViews(['sources', 'datasets']);
310
+ }).catch(
311
+ (pError) =>
312
+ {
313
+ this.pict.views['Pict-Section-Modal'].toast('Error: ' + pError.message, {type: 'error'});
314
+ });
315
+ }
316
+
317
+ async provisionSelected()
318
+ {
319
+ let tmpSelected = this.getSelectedFolderNames();
320
+ if (tmpSelected.length === 0)
321
+ {
322
+ this.pict.views['Pict-Section-Modal'].toast('Select datasets to provision using the checkboxes', {type: 'warning'});
323
+ return;
324
+ }
325
+
326
+ let tmpConfirmed = await this.pict.views['Pict-Section-Modal'].confirm('Provision ' + tmpSelected.length + ' selected dataset(s)?', { title: 'Provision Selected', confirmLabel: 'Provision' });
327
+ if (!tmpConfirmed) return;
328
+
329
+ this.provisionBatch(tmpSelected, 0, 0, 0);
330
+ }
331
+
332
+ async provisionAll()
333
+ {
334
+ let tmpConfirmed = await this.pict.views['Pict-Section-Modal'].confirm('Provision ALL discovered datasets?', { title: 'Provision All', confirmLabel: 'Provision All', dangerous: true });
335
+ if (!tmpConfirmed) return;
336
+
337
+ this.pict.views['Pict-Section-Modal'].toast('Provisioning all datasets...', {type: 'info'});
338
+
339
+ this.pict.providers.Facto.provisionAllScannerDatasets().then(
340
+ (pResponse) =>
341
+ {
342
+ if (pResponse && pResponse.Error)
343
+ {
344
+ this.pict.views['Pict-Section-Modal'].toast('Error: ' + pResponse.Error, {type: 'error'});
345
+ return;
346
+ }
347
+ this.pict.views['Pict-Section-Modal'].toast('Provisioned ' + pResponse.Provisioned + ' of ' + pResponse.Total + ' (' + pResponse.Errors + ' error(s))', {type: 'success'});
348
+ this.loadScannerState();
349
+ this.pict.providers.FactoUI.refreshDataViews(['sources', 'datasets']);
350
+ }).catch(
351
+ (pError) =>
352
+ {
353
+ this.pict.views['Pict-Section-Modal'].toast('Error: ' + pError.message, {type: 'error'});
354
+ });
355
+ }
356
+
357
+ provisionBatch(pFolderNames, pIndex, pSuccessCount, pErrorCount)
358
+ {
359
+ if (pIndex >= pFolderNames.length)
360
+ {
361
+ this.pict.views['Pict-Section-Modal'].toast('Provisioned ' + pSuccessCount + ' of ' + pFolderNames.length + ' (' + pErrorCount + ' error(s))', {type: 'success'});
362
+ this.loadScannerState();
363
+ this.pict.providers.FactoUI.refreshDataViews(['sources', 'datasets']);
364
+ return;
365
+ }
366
+
367
+ let tmpName = pFolderNames[pIndex];
368
+ this.pict.views['Pict-Section-Modal'].toast('Provisioning ' + (pIndex + 1) + '/' + pFolderNames.length + ': ' + tmpName + '...', {type: 'info'});
369
+
370
+ this.pict.providers.Facto.provisionScannerDataset(tmpName).then(
371
+ (pResponse) =>
372
+ {
373
+ if (pResponse && pResponse.Error)
374
+ {
375
+ this.provisionBatch(pFolderNames, pIndex + 1, pSuccessCount, pErrorCount + 1);
376
+ }
377
+ else
378
+ {
379
+ this.provisionBatch(pFolderNames, pIndex + 1, pSuccessCount + 1, pErrorCount);
380
+ }
381
+ }).catch(
382
+ () =>
383
+ {
384
+ this.provisionBatch(pFolderNames, pIndex + 1, pSuccessCount, pErrorCount + 1);
385
+ });
386
+ }
387
+
388
+ toggleRowMenu(pEvent, pFolderName)
389
+ {
390
+ pEvent.stopPropagation();
391
+ let tmpEl = document.getElementById('facto-row-actions-' + pFolderName);
392
+ if (!tmpEl) return;
393
+
394
+ let tmpWasOpen = tmpEl.classList.contains('open');
395
+ this.closeRowMenus();
396
+
397
+ if (!tmpWasOpen)
398
+ {
399
+ tmpEl.classList.add('open');
400
+ let tmpCloseHandler = (pCloseEvent) =>
401
+ {
402
+ if (!tmpEl.contains(pCloseEvent.target))
403
+ {
404
+ tmpEl.classList.remove('open');
405
+ document.removeEventListener('click', tmpCloseHandler);
406
+ }
407
+ };
408
+ setTimeout(() => { document.addEventListener('click', tmpCloseHandler); }, 0);
409
+ }
410
+ }
411
+
412
+ closeRowMenus()
413
+ {
414
+ let tmpOpenMenus = document.querySelectorAll('.facto-row-actions.open');
415
+ for (let i = 0; i < tmpOpenMenus.length; i++)
416
+ {
417
+ tmpOpenMenus[i].classList.remove('open');
418
+ }
419
+ }
420
+
421
+ viewDetail(pFolderName)
422
+ {
423
+ let tmpContainer = document.getElementById('facto-scanner-detail');
424
+ if (!tmpContainer) return;
425
+
426
+ tmpContainer.innerHTML = '<p style="color:#888;">Loading details for ' + this.escapeHtml(pFolderName) + '...</p>';
427
+
428
+ this.pict.providers.Facto.loadScannerDatasetDetail(pFolderName).then(
429
+ (pDS) =>
430
+ {
431
+ if (pDS && pDS.Error)
432
+ {
433
+ tmpContainer.innerHTML = '<p style="color:#dc3545;">' + pDS.Error + '</p>';
434
+ return;
435
+ }
436
+
437
+ let tmpHtml = '<div style="border:1px solid #ddd; border-radius:6px; padding:16px; background:#fafafa;">';
438
+ tmpHtml += '<div style="display:flex; justify-content:space-between; align-items:center; margin-bottom:12px;">';
439
+ tmpHtml += '<h3 style="margin:0; font-size:1.1em; color:#333;">' + this.escapeHtml(pDS.Title || pDS.FolderName) + '</h3>';
440
+ tmpHtml += '<button class="secondary" style="padding:4px 10px; font-size:0.82em;" onclick="document.getElementById(\'facto-scanner-detail\').innerHTML=\'\';">Close</button>';
441
+ tmpHtml += '</div>';
442
+
443
+ // Metadata grid
444
+ tmpHtml += '<div style="display:grid; grid-template-columns:1fr 1fr; gap:8px 20px; font-size:0.9em; margin-bottom:12px;">';
445
+ tmpHtml += '<div><strong>Folder:</strong> <span style="font-family:monospace;">' + this.escapeHtml(pDS.FolderName) + '</span></div>';
446
+ tmpHtml += '<div><strong>Status:</strong> <span class="badge ' + this.getStatusBadgeClass(pDS.Status) + '">' + (pDS.Status || '') + '</span></div>';
447
+ tmpHtml += '<div><strong>Provider:</strong> ' + this.escapeHtml(pDS.Provider || 'Unknown') + '</div>';
448
+ tmpHtml += '<div><strong>License:</strong> ' + this.escapeHtml(pDS.License || 'Unknown') + '</div>';
449
+ if (pDS.SourceURL)
450
+ {
451
+ tmpHtml += '<div><strong>Source:</strong> <a href="' + this.escapeHtml(pDS.SourceURL) + '" target="_blank">' + this.escapeHtml(pDS.SourceURL.substring(0, 60)) + '</a></div>';
452
+ }
453
+ if (pDS.UpdateFrequency)
454
+ {
455
+ tmpHtml += '<div><strong>Update Frequency:</strong> ' + this.escapeHtml(pDS.UpdateFrequency.substring(0, 100)) + '</div>';
456
+ }
457
+ if (pDS.RecordCount)
458
+ {
459
+ tmpHtml += '<div><strong>Record Count:</strong> ' + this.escapeHtml(pDS.RecordCount.substring(0, 100)) + '</div>';
460
+ }
461
+ if (pDS.IDSource)
462
+ {
463
+ tmpHtml += '<div><strong>Source ID:</strong> #' + pDS.IDSource + '</div>';
464
+ }
465
+ if (pDS.IDDataset)
466
+ {
467
+ tmpHtml += '<div><strong>Dataset ID:</strong> #' + pDS.IDDataset + '</div>';
468
+ }
469
+ tmpHtml += '</div>';
470
+
471
+ // Description
472
+ if (pDS.Description)
473
+ {
474
+ tmpHtml += '<div style="margin-bottom:12px;"><strong>Description:</strong><div style="color:#555; font-size:0.9em; margin-top:4px; max-height:100px; overflow-y:auto; white-space:pre-wrap;">' + this.escapeHtml(pDS.Description.substring(0, 500)) + '</div></div>';
475
+ }
476
+
477
+ // Data files
478
+ if (pDS.DataFiles && pDS.DataFiles.length > 0)
479
+ {
480
+ tmpHtml += '<div style="margin-bottom:12px;"><strong>Data Files (' + pDS.DataFiles.length + '):</strong>';
481
+ tmpHtml += '<table style="margin-top:4px; font-size:0.85em;"><thead><tr><th>File</th><th>Format</th><th>Size</th><th>Compressed</th></tr></thead><tbody>';
482
+ for (let i = 0; i < pDS.DataFiles.length && i < 20; i++)
483
+ {
484
+ let tmpFile = pDS.DataFiles[i];
485
+ let tmpSize = this.formatSize(tmpFile.Size || 0);
486
+ tmpHtml += '<tr>';
487
+ tmpHtml += '<td style="font-family:monospace; font-size:0.9em;">' + this.escapeHtml(tmpFile.FileName) + '</td>';
488
+ tmpHtml += '<td>' + (tmpFile.Format || 'unknown') + '</td>';
489
+ tmpHtml += '<td>' + tmpSize + '</td>';
490
+ tmpHtml += '<td>' + (tmpFile.Compressed ? 'Yes' : 'No') + '</td>';
491
+ tmpHtml += '</tr>';
492
+ }
493
+ if (pDS.DataFiles.length > 20)
494
+ {
495
+ tmpHtml += '<tr><td colspan="4" style="color:#888;">...and ' + (pDS.DataFiles.length - 20) + ' more file(s)</td></tr>';
496
+ }
497
+ tmpHtml += '</tbody></table></div>';
498
+ }
499
+ else
500
+ {
501
+ tmpHtml += '<p style="color:#dc3545; font-size:0.9em; margin-bottom:12px;"><strong>No data files.</strong> Download needed.</p>';
502
+ }
503
+
504
+ // Errors
505
+ if (pDS.Errors && pDS.Errors.length > 0)
506
+ {
507
+ tmpHtml += '<div style="margin-bottom:12px; padding:8px; background:#f8d7da; border-radius:4px; font-size:0.85em; color:#721c24;"><strong>Errors:</strong><ul style="margin:4px 0 0 16px;">';
508
+ for (let i = 0; i < pDS.Errors.length; i++)
509
+ {
510
+ tmpHtml += '<li>' + this.escapeHtml(pDS.Errors[i]) + '</li>';
511
+ }
512
+ tmpHtml += '</ul></div>';
513
+ }
514
+
515
+ // Action buttons
516
+ tmpHtml += '<div>';
517
+ if (pDS.Status === 'Discovered')
518
+ {
519
+ tmpHtml += '<button class="success" onclick="pict.views[\'Facto-Scanner\'].provisionOne(\'' + this.escapeAttr(pDS.FolderName) + '\')">Provision</button>';
520
+ }
521
+ tmpHtml += '</div>';
522
+
523
+ tmpHtml += '</div>';
524
+ tmpContainer.innerHTML = tmpHtml;
525
+ }).catch(
526
+ (pError) =>
527
+ {
528
+ tmpContainer.innerHTML = '<p style="color:#dc3545;">Error: ' + pError.message + '</p>';
529
+ });
530
+ }
531
+
532
+ formatSize(pBytes)
533
+ {
534
+ if (pBytes === 0) return '0 B';
535
+ let tmpUnits = ['B', 'KB', 'MB', 'GB', 'TB'];
536
+ let tmpIndex = Math.floor(Math.log(pBytes) / Math.log(1024));
537
+ return (pBytes / Math.pow(1024, tmpIndex)).toFixed(1) + ' ' + tmpUnits[tmpIndex];
538
+ }
539
+
540
+
541
+ }
542
+
543
+ module.exports = FactoScannerView;
544
+
545
+ module.exports.default_configuration =
546
+ {
547
+ ViewIdentifier: 'Facto-Scanner',
548
+ DefaultRenderable: 'Facto-Scanner',
549
+ DefaultDestinationAddress: '#Facto-Section-Scanner',
550
+ Templates:
551
+ [
552
+ {
553
+ Hash: 'Facto-Scanner',
554
+ Template: /*html*/`
555
+ <div class="accordion-row">
556
+ <div class="accordion-number">&#9776;</div>
557
+ <div class="accordion-card open" id="facto-card-scanner">
558
+ <div class="accordion-header" onclick="pict.views['Facto-Layout'].toggleSection('facto-card-scanner')">
559
+ <span class="accordion-title">Source Folder Scanner</span>
560
+ <span class="accordion-preview">Discover and provision datasets from folder trees</span>
561
+ <span class="accordion-toggle">&#9660;</span>
562
+ </div>
563
+ <div class="accordion-body">
564
+ <p style="margin-bottom:12px; color:#666; font-size:0.9em;">Point the scanner at folder trees containing dataset research (README.md + data/). Discovered datasets can be provisioned into the database individually or in bulk.</p>
565
+
566
+ <!-- Summary -->
567
+ <div id="facto-scanner-summary" style="margin-bottom:12px; padding:8px 12px; background:#e9ecef; border-radius:4px; font-size:0.9em; color:#555;">
568
+ No datasets loaded
569
+ </div>
570
+
571
+ <!-- Add scan path -->
572
+ <div class="inline-group" style="margin-bottom:12px;">
573
+ <div style="flex:3;">
574
+ <input type="text" id="facto-scanner-path-input" placeholder="/path/to/dataset/folder/tree" style="margin-bottom:0;">
575
+ </div>
576
+ <div style="flex:0 0 auto; display:flex; align-items:flex-end; gap:4px;">
577
+ <button class="primary" style="margin-bottom:0;" onclick="pict.views['Facto-Scanner'].addPath()">Add &amp; Scan</button>
578
+ <button class="secondary" style="margin-bottom:0;" onclick="pict.views['Facto-Scanner'].rescanAll()">Re-scan All</button>
579
+ </div>
580
+ </div>
581
+
582
+ <!-- Scan paths list -->
583
+ <div id="facto-scanner-paths-list" style="margin-bottom:16px;"></div>
584
+
585
+ <!-- Filter bar -->
586
+ <div class="inline-group" style="margin-bottom:8px;">
587
+ <div style="flex:3;">
588
+ <input type="text" id="facto-scanner-search" placeholder="Search by name, title, or provider..." style="margin-bottom:0;" oninput="pict.views['Facto-Scanner'].filterDatasets()">
589
+ </div>
590
+ <div style="flex:1;">
591
+ <select id="facto-scanner-status-filter" style="margin-bottom:0;" onchange="pict.views['Facto-Scanner'].filterDatasets()">
592
+ <option value="">All Statuses</option>
593
+ <option value="Discovered">Discovered</option>
594
+ <option value="Provisioned">Provisioned</option>
595
+ <option value="Ingested">Ingested</option>
596
+ <option value="Error">Error</option>
597
+ </select>
598
+ </div>
599
+ <div style="flex:0 0 auto; display:flex; align-items:flex-end; gap:4px;">
600
+ <button class="success" style="margin-bottom:0;" onclick="pict.views['Facto-Scanner'].provisionSelected()">Provision Selected</button>
601
+ <button class="success" style="margin-bottom:0;" onclick="pict.views['Facto-Scanner'].provisionAll()">Provision All</button>
602
+ </div>
603
+ </div>
604
+
605
+ <!-- Datasets table -->
606
+ <div id="facto-scanner-datasets-list"></div>
607
+
608
+ <!-- Detail panel -->
609
+ <div id="facto-scanner-detail" style="margin-top:12px;"></div>
610
+ </div>
611
+ </div>
612
+ </div>
613
+ `
614
+ }
615
+ ],
616
+ Renderables:
617
+ [
618
+ {
619
+ RenderableHash: 'Facto-Scanner',
620
+ TemplateHash: 'Facto-Scanner',
621
+ DestinationAddress: '#Facto-Section-Scanner'
622
+ }
623
+ ]
624
+ };