retold-facto 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/.claude/launch.json +11 -0
  2. package/.dockerignore +8 -0
  3. package/.quackage.json +19 -0
  4. package/Dockerfile +26 -0
  5. package/bin/retold-facto.js +909 -0
  6. package/examples/facto-government-data.sqlite +0 -0
  7. package/examples/government-data-catalog.json +137 -0
  8. package/examples/government-data-loader.js +1432 -0
  9. package/package.json +91 -0
  10. package/scripts/facto-download.js +425 -0
  11. package/source/Retold-Facto.js +1042 -0
  12. package/source/services/Retold-Facto-BeaconProvider.js +511 -0
  13. package/source/services/Retold-Facto-CatalogManager.js +1252 -0
  14. package/source/services/Retold-Facto-DataLakeService.js +1642 -0
  15. package/source/services/Retold-Facto-DatasetManager.js +417 -0
  16. package/source/services/Retold-Facto-IngestEngine.js +1315 -0
  17. package/source/services/Retold-Facto-ProjectionEngine.js +3960 -0
  18. package/source/services/Retold-Facto-RecordManager.js +360 -0
  19. package/source/services/Retold-Facto-SchemaManager.js +1110 -0
  20. package/source/services/Retold-Facto-SourceFolderScanner.js +2243 -0
  21. package/source/services/Retold-Facto-SourceManager.js +730 -0
  22. package/source/services/Retold-Facto-StoreConnectionManager.js +441 -0
  23. package/source/services/Retold-Facto-ThroughputMonitor.js +478 -0
  24. package/source/services/web-app/codemirror-entry.js +7 -0
  25. package/source/services/web-app/pict-app/Pict-Application-Facto-Configuration.json +9 -0
  26. package/source/services/web-app/pict-app/Pict-Application-Facto.js +70 -0
  27. package/source/services/web-app/pict-app/Pict-Facto-Bundle.js +11 -0
  28. package/source/services/web-app/pict-app/providers/Pict-Provider-Facto-UI.js +66 -0
  29. package/source/services/web-app/pict-app/providers/Pict-Provider-Facto.js +69 -0
  30. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Catalog.js +93 -0
  31. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Connections.js +42 -0
  32. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Datasets.js +605 -0
  33. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Projections.js +188 -0
  34. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Scanner.js +80 -0
  35. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Schema.js +116 -0
  36. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Sources.js +104 -0
  37. package/source/services/web-app/pict-app/views/PictView-Facto-Catalog.js +526 -0
  38. package/source/services/web-app/pict-app/views/PictView-Facto-Datasets.js +173 -0
  39. package/source/services/web-app/pict-app/views/PictView-Facto-Ingest.js +259 -0
  40. package/source/services/web-app/pict-app/views/PictView-Facto-Layout.js +191 -0
  41. package/source/services/web-app/pict-app/views/PictView-Facto-Projections.js +231 -0
  42. package/source/services/web-app/pict-app/views/PictView-Facto-Records.js +326 -0
  43. package/source/services/web-app/pict-app/views/PictView-Facto-Scanner.js +624 -0
  44. package/source/services/web-app/pict-app/views/PictView-Facto-Sources.js +201 -0
  45. package/source/services/web-app/pict-app/views/PictView-Facto-Throughput.js +456 -0
  46. package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full-Configuration.json +14 -0
  47. package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full.js +391 -0
  48. package/source/services/web-app/pict-app-full/providers/PictRouter-Facto-Configuration.json +56 -0
  49. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-BottomBar.js +68 -0
  50. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Connections.js +340 -0
  51. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboard.js +149 -0
  52. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboards.js +819 -0
  53. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Datasets.js +178 -0
  54. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-IngestJobs.js +99 -0
  55. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Layout.js +62 -0
  56. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-MappingEditor.js +158 -0
  57. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-ProjectionDetail.js +1120 -0
  58. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Projections.js +172 -0
  59. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-QueryPanel.js +119 -0
  60. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-RecordViewer.js +663 -0
  61. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Records.js +648 -0
  62. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Scanner.js +1017 -0
  63. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDetail.js +1404 -0
  64. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDocEditor.js +1036 -0
  65. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaEditor.js +636 -0
  66. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaResearch.js +357 -0
  67. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceDetail.js +822 -0
  68. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceEditor.js +1036 -0
  69. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceResearch.js +487 -0
  70. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Sources.js +165 -0
  71. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Throughput.js +439 -0
  72. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-TopBar.js +335 -0
  73. package/source/services/web-app/pict-app-full/views/projections/Facto-Projections-Constants.js +71 -0
  74. package/source/services/web-app/web/chart.min.js +20 -0
  75. package/source/services/web-app/web/codemirror-bundle.js +30099 -0
  76. package/source/services/web-app/web/css/facto-themes.css +467 -0
  77. package/source/services/web-app/web/css/facto.css +502 -0
  78. package/source/services/web-app/web/index.html +28 -0
  79. package/source/services/web-app/web/retold-facto.js +12138 -0
  80. package/source/services/web-app/web/retold-facto.js.map +1 -0
  81. package/source/services/web-app/web/retold-facto.min.js +2 -0
  82. package/source/services/web-app/web/retold-facto.min.js.map +1 -0
  83. package/source/services/web-app/web/simple/index.html +17 -0
  84. package/test/Facto_Browser_Integration_tests.js +798 -0
  85. package/test/RetoldFacto_tests.js +4117 -0
  86. package/test/fixtures/weather-readings.csv +17 -0
  87. package/test/fixtures/weather-stations.csv +9 -0
  88. package/test/model/MeadowModel-Extended.json +8497 -0
  89. package/test/model/MeadowModel-PICT.json +1 -0
  90. package/test/model/MeadowModel.json +1355 -0
  91. package/test/model/ddl/Facto.ddl +225 -0
  92. package/test/model/fable-configuration.json +14 -0
@@ -0,0 +1,357 @@
1
+ const libPictView = require('pict-view');
2
+
3
+ const _ViewConfiguration =
4
+ {
5
+ ViewIdentifier: "Facto-Full-SchemaResearch",
6
+
7
+ DefaultRenderable: "Facto-Full-SchemaResearch-Content",
8
+ DefaultDestinationAddress: "#Facto-Full-Content-Container",
9
+
10
+ AutoRender: false,
11
+
12
+ CSS: /*css*/`
13
+ .facto-schema-research-search {
14
+ display: flex;
15
+ gap: 0.75em;
16
+ margin-bottom: 1.25em;
17
+ }
18
+ .facto-schema-research-search input {
19
+ flex: 1;
20
+ margin-bottom: 0;
21
+ }
22
+ .facto-schema-research-add-form {
23
+ background: var(--facto-bg-card);
24
+ border: 1px solid var(--facto-border);
25
+ border-radius: 8px;
26
+ padding: 1.25em;
27
+ margin-bottom: 1.25em;
28
+ }
29
+ .facto-schema-research-add-form .facto-form-grid {
30
+ display: grid;
31
+ grid-template-columns: 1fr 1fr;
32
+ gap: 0.75em;
33
+ }
34
+ .facto-schema-research-add-form .facto-form-grid .facto-form-full {
35
+ grid-column: 1 / -1;
36
+ }
37
+ .facto-schema-research-add-form label {
38
+ display: block;
39
+ font-size: 0.8em;
40
+ font-weight: 600;
41
+ margin-bottom: 0.25em;
42
+ color: var(--facto-text-muted);
43
+ }
44
+ .facto-schema-research-add-form input,
45
+ .facto-schema-research-add-form textarea,
46
+ .facto-schema-research-add-form select {
47
+ width: 100%;
48
+ margin-bottom: 0;
49
+ }
50
+ .facto-schema-research-add-form textarea {
51
+ font-size: 0.9em;
52
+ }
53
+ .facto-schema-research-add-form .facto-form-actions {
54
+ margin-top: 1em;
55
+ display: flex;
56
+ gap: 0.5em;
57
+ }
58
+ .facto-schema-research-header-row {
59
+ display: flex;
60
+ justify-content: space-between;
61
+ align-items: flex-start;
62
+ margin-bottom: 0;
63
+ }
64
+ .facto-schema-research-import textarea {
65
+ width: 100%;
66
+ font-family: 'SF Mono', Consolas, monospace;
67
+ font-size: 0.85em;
68
+ padding: 0.75em;
69
+ background: var(--facto-bg-input);
70
+ color: var(--facto-text);
71
+ border: 1px solid var(--facto-border);
72
+ border-radius: 6px;
73
+ margin-bottom: 0.5em;
74
+ }
75
+ `,
76
+
77
+ Templates:
78
+ [
79
+ {
80
+ Hash: "Facto-Full-SchemaResearch-Template",
81
+ Template: /*html*/`
82
+ <div class="facto-content">
83
+ <div class="facto-content-header facto-schema-research-header-row">
84
+ <div>
85
+ <h1>Schema Research</h1>
86
+ <p>Research, define, and document schemas that describe the shape of records in your data pools.</p>
87
+ </div>
88
+ <button class="facto-btn facto-btn-success" onclick="{~P~}.views['Facto-Full-SchemaResearch'].toggleAddForm()">+ Add Schema</button>
89
+ </div>
90
+
91
+ <div id="Facto-Full-SchemaResearch-AddForm" style="display:none;">
92
+ <div class="facto-schema-research-add-form">
93
+ <div class="facto-form-grid">
94
+ <div>
95
+ <label>Name</label>
96
+ <input type="text" id="Facto-SchemaResearch-Add-Name" placeholder="Schema name">
97
+ </div>
98
+ <div>
99
+ <label>Type</label>
100
+ <select id="Facto-SchemaResearch-Add-Type">
101
+ <option value="Record">Record</option>
102
+ <option value="Event">Event</option>
103
+ <option value="Document">Document</option>
104
+ <option value="Metric">Metric</option>
105
+ <option value="Reference">Reference</option>
106
+ <option value="Other">Other</option>
107
+ </select>
108
+ </div>
109
+ <div class="facto-form-full">
110
+ <label>Description</label>
111
+ <textarea id="Facto-SchemaResearch-Add-Description" rows="2" placeholder="Brief description of what this schema defines"></textarea>
112
+ </div>
113
+ <div class="facto-form-full">
114
+ <label>Schema Definition (MicroDDL) — optional, can be added later</label>
115
+ <textarea id="Facto-SchemaResearch-Add-DDL" rows="4" placeholder="!TableName&#10;@IDColumn&#10;$Name 200&#10;#Count&#10;&CreatedDate" style="font-family: 'SF Mono', Consolas, monospace; font-size: 0.85em;"></textarea>
116
+ </div>
117
+ </div>
118
+ <div class="facto-form-actions">
119
+ <button class="facto-btn facto-btn-primary" onclick="{~P~}.views['Facto-Full-SchemaResearch'].createEntry()">Save Schema</button>
120
+ <button class="facto-btn facto-btn-secondary" onclick="{~P~}.views['Facto-Full-SchemaResearch'].toggleAddForm()">Cancel</button>
121
+ </div>
122
+ </div>
123
+ </div>
124
+
125
+ <div class="facto-schema-research-search">
126
+ <input type="text" id="Facto-Full-SchemaResearch-Search" placeholder="Search schemas by name, type, or description...">
127
+ <button class="facto-btn facto-btn-primary" onclick="{~P~}.views['Facto-Full-SchemaResearch'].searchSchemas()">Search</button>
128
+ </div>
129
+
130
+ <div id="Facto-Full-SchemaResearch-List"></div>
131
+ <div id="Facto-Full-SchemaResearch-Status" class="facto-status" style="display:none;"></div>
132
+
133
+ <div class="facto-section" style="margin-top:2em;">
134
+ <div class="facto-section-title">Import / Export</div>
135
+ <div class="facto-schema-research-import">
136
+ <textarea id="Facto-Full-SchemaResearch-ImportJSON" rows="4" placeholder="Paste JSON array of schema definitions here..."></textarea>
137
+ <button class="facto-btn facto-btn-primary" onclick="{~P~}.views['Facto-Full-SchemaResearch'].importSchemas()">Import JSON</button>
138
+ <button class="facto-btn facto-btn-secondary" onclick="{~P~}.views['Facto-Full-SchemaResearch'].exportSchemas()">Export Schemas</button>
139
+ </div>
140
+ </div>
141
+ </div>
142
+ `
143
+ }
144
+ ],
145
+
146
+ Renderables:
147
+ [
148
+ {
149
+ RenderableHash: "Facto-Full-SchemaResearch-Content",
150
+ TemplateHash: "Facto-Full-SchemaResearch-Template",
151
+ DestinationAddress: "#Facto-Full-Content-Container",
152
+ RenderMethod: "replace"
153
+ }
154
+ ]
155
+ };
156
+
157
+ class FactoFullSchemaResearchView extends libPictView
158
+ {
159
+ constructor(pFable, pOptions, pServiceHash)
160
+ {
161
+ super(pFable, pOptions, pServiceHash);
162
+ }
163
+
164
+ onAfterRender(pRenderable, pRenderDestinationAddress, pRecord, pContent)
165
+ {
166
+ this.pict.providers.Facto.loadSchemas().then(
167
+ () =>
168
+ {
169
+ this.refreshList();
170
+ });
171
+
172
+ return super.onAfterRender(pRenderable, pRenderDestinationAddress, pRecord, pContent);
173
+ }
174
+
175
+ setStatus(pMessage, pType)
176
+ {
177
+ let tmpEl = document.getElementById('Facto-Full-SchemaResearch-Status');
178
+ if (!tmpEl) return;
179
+ tmpEl.className = 'facto-status facto-status-' + (pType || 'info');
180
+ tmpEl.textContent = pMessage;
181
+ tmpEl.style.display = 'block';
182
+ }
183
+
184
+ refreshList()
185
+ {
186
+ let tmpContainer = document.getElementById('Facto-Full-SchemaResearch-List');
187
+ if (!tmpContainer) return;
188
+
189
+ let tmpSchemas = this.pict.AppData.Facto.Schemas;
190
+ if (!tmpSchemas || tmpSchemas.length === 0)
191
+ {
192
+ tmpContainer.innerHTML = '<div class="facto-empty">No schemas yet. Add a schema to begin defining your data shapes.</div>';
193
+ return;
194
+ }
195
+
196
+ let tmpHtml = '<table><thead><tr><th>ID</th><th>Name</th><th>Type</th><th>Version</th><th>Hash</th><th>Active</th><th>Actions</th></tr></thead><tbody>';
197
+ for (let i = 0; i < tmpSchemas.length; i++)
198
+ {
199
+ let tmpSchema = tmpSchemas[i];
200
+ let tmpActive = tmpSchema.Active ? '<span class="facto-badge facto-badge-success">Active</span>' : '<span class="facto-badge facto-badge-muted">Inactive</span>';
201
+ let tmpHash = tmpSchema.SchemaHash || '';
202
+ if (tmpHash.length > 12) tmpHash = tmpHash.substring(0, 12) + '\u2026';
203
+ tmpHtml += '<tr>';
204
+ tmpHtml += '<td>' + (tmpSchema.IDSchema || '') + '</td>';
205
+ tmpHtml += '<td>' + (tmpSchema.Name || '') + '</td>';
206
+ tmpHtml += '<td><span class="facto-badge facto-badge-primary">' + (tmpSchema.Type || '') + '</span></td>';
207
+ tmpHtml += '<td>' + (tmpSchema.Version || 0) + '</td>';
208
+ tmpHtml += '<td><code style="font-size:0.8em;">' + tmpHash + '</code></td>';
209
+ tmpHtml += '<td>' + tmpActive + '</td>';
210
+ tmpHtml += '<td>';
211
+ tmpHtml += '<button class="facto-btn facto-btn-primary facto-btn-small" onclick="pict.PictApplication.navigateTo(\'/Schema/' + tmpSchema.IDSchema + '\')">View</button> ';
212
+ tmpHtml += '<button class="facto-btn facto-btn-danger facto-btn-small" onclick="pict.views[\'Facto-Full-SchemaResearch\'].deleteEntry(' + tmpSchema.IDSchema + ')">Delete</button>';
213
+ tmpHtml += '</td>';
214
+ tmpHtml += '</tr>';
215
+ }
216
+ tmpHtml += '</tbody></table>';
217
+ tmpContainer.innerHTML = tmpHtml;
218
+ }
219
+
220
+ toggleAddForm()
221
+ {
222
+ let tmpForm = document.getElementById('Facto-Full-SchemaResearch-AddForm');
223
+ if (!tmpForm) return;
224
+ tmpForm.style.display = (tmpForm.style.display === 'none') ? 'block' : 'none';
225
+ }
226
+
227
+ createEntry()
228
+ {
229
+ let tmpData =
230
+ {
231
+ Name: (document.getElementById('Facto-SchemaResearch-Add-Name') || {}).value || '',
232
+ Type: (document.getElementById('Facto-SchemaResearch-Add-Type') || {}).value || 'Record',
233
+ Description: (document.getElementById('Facto-SchemaResearch-Add-Description') || {}).value || '',
234
+ SchemaDefinition: (document.getElementById('Facto-SchemaResearch-Add-DDL') || {}).value || ''
235
+ };
236
+
237
+ if (!tmpData.Name)
238
+ {
239
+ this.pict.providers.FactoUI.showToast('Name is required', 'error');
240
+ return;
241
+ }
242
+
243
+ this.pict.providers.Facto.createSchema(tmpData).then(
244
+ (pResponse) =>
245
+ {
246
+ if (pResponse && pResponse.Error)
247
+ {
248
+ this.pict.providers.FactoUI.showToast('Error: ' + pResponse.Error, 'error');
249
+ return;
250
+ }
251
+
252
+ this.pict.providers.FactoUI.showToast('Schema created', 'ok');
253
+
254
+ // Clear the form fields
255
+ let tmpFields = ['Name', 'Description', 'DDL'];
256
+ for (let i = 0; i < tmpFields.length; i++)
257
+ {
258
+ let tmpEl = document.getElementById('Facto-SchemaResearch-Add-' + tmpFields[i]);
259
+ if (tmpEl) tmpEl.value = '';
260
+ }
261
+ let tmpTypeEl = document.getElementById('Facto-SchemaResearch-Add-Type');
262
+ if (tmpTypeEl) tmpTypeEl.selectedIndex = 0;
263
+
264
+ // Hide the form and refresh the list
265
+ let tmpForm = document.getElementById('Facto-Full-SchemaResearch-AddForm');
266
+ if (tmpForm) tmpForm.style.display = 'none';
267
+
268
+ return this.pict.providers.Facto.loadSchemas();
269
+ }).then(
270
+ () =>
271
+ {
272
+ this.refreshList();
273
+ });
274
+ }
275
+
276
+ searchSchemas()
277
+ {
278
+ let tmpQuery = (document.getElementById('Facto-Full-SchemaResearch-Search') || {}).value || '';
279
+ if (!tmpQuery)
280
+ {
281
+ this.pict.providers.Facto.loadSchemas().then(
282
+ () => { this.refreshList(); });
283
+ return;
284
+ }
285
+
286
+ // Client-side filter on loaded schemas
287
+ let tmpLower = tmpQuery.toLowerCase();
288
+ let tmpAll = this.pict.AppData.Facto.Schemas || [];
289
+ this.pict.AppData.Facto.Schemas = tmpAll.filter(
290
+ (pSchema) =>
291
+ {
292
+ return (pSchema.Name && pSchema.Name.toLowerCase().indexOf(tmpLower) >= 0) ||
293
+ (pSchema.Type && pSchema.Type.toLowerCase().indexOf(tmpLower) >= 0) ||
294
+ (pSchema.Description && pSchema.Description.toLowerCase().indexOf(tmpLower) >= 0);
295
+ });
296
+ this.refreshList();
297
+ // Restore full list in AppData for next time
298
+ this.pict.AppData.Facto.Schemas = tmpAll;
299
+ }
300
+
301
+ deleteEntry(pIDSchema)
302
+ {
303
+ if (!confirm('Delete this schema?')) return;
304
+ this.pict.providers.Facto.api('DELETE', '/1.0/Schema/' + pIDSchema).then(
305
+ () => { return this.pict.providers.Facto.loadSchemas(); }).then(
306
+ () => { this.refreshList(); this.setStatus('Schema removed', 'ok'); });
307
+ }
308
+
309
+ importSchemas()
310
+ {
311
+ let tmpTextArea = document.getElementById('Facto-Full-SchemaResearch-ImportJSON');
312
+ if (!tmpTextArea || !tmpTextArea.value)
313
+ {
314
+ this.setStatus('Paste JSON to import', 'warn');
315
+ return;
316
+ }
317
+
318
+ let tmpSchemas;
319
+ try { tmpSchemas = JSON.parse(tmpTextArea.value); }
320
+ catch (pErr) { this.setStatus('Invalid JSON: ' + pErr.message, 'error'); return; }
321
+
322
+ if (!Array.isArray(tmpSchemas))
323
+ {
324
+ tmpSchemas = [tmpSchemas];
325
+ }
326
+
327
+ let tmpPromises = tmpSchemas.map(
328
+ (pSchema) =>
329
+ {
330
+ return this.pict.providers.Facto.createSchema(pSchema);
331
+ });
332
+
333
+ Promise.all(tmpPromises).then(
334
+ () =>
335
+ {
336
+ this.setStatus('Imported ' + tmpSchemas.length + ' schema(s)', 'ok');
337
+ tmpTextArea.value = '';
338
+ return this.pict.providers.Facto.loadSchemas();
339
+ }).then(
340
+ () => { this.refreshList(); });
341
+ }
342
+
343
+ exportSchemas()
344
+ {
345
+ let tmpTextArea = document.getElementById('Facto-Full-SchemaResearch-ImportJSON');
346
+ if (tmpTextArea)
347
+ {
348
+ let tmpSchemas = this.pict.AppData.Facto.Schemas || [];
349
+ tmpTextArea.value = JSON.stringify(tmpSchemas, null, 2);
350
+ }
351
+ this.setStatus('Schemas exported to JSON text area', 'ok');
352
+ }
353
+ }
354
+
355
+ module.exports = FactoFullSchemaResearchView;
356
+
357
+ module.exports.default_configuration = _ViewConfiguration;