retold-facto 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/launch.json +11 -0
- package/.dockerignore +8 -0
- package/.quackage.json +19 -0
- package/Dockerfile +26 -0
- package/bin/retold-facto.js +909 -0
- package/examples/facto-government-data.sqlite +0 -0
- package/examples/government-data-catalog.json +137 -0
- package/examples/government-data-loader.js +1432 -0
- package/package.json +91 -0
- package/scripts/facto-download.js +425 -0
- package/source/Retold-Facto.js +1042 -0
- package/source/services/Retold-Facto-BeaconProvider.js +511 -0
- package/source/services/Retold-Facto-CatalogManager.js +1252 -0
- package/source/services/Retold-Facto-DataLakeService.js +1642 -0
- package/source/services/Retold-Facto-DatasetManager.js +417 -0
- package/source/services/Retold-Facto-IngestEngine.js +1315 -0
- package/source/services/Retold-Facto-ProjectionEngine.js +3960 -0
- package/source/services/Retold-Facto-RecordManager.js +360 -0
- package/source/services/Retold-Facto-SchemaManager.js +1110 -0
- package/source/services/Retold-Facto-SourceFolderScanner.js +2243 -0
- package/source/services/Retold-Facto-SourceManager.js +730 -0
- package/source/services/Retold-Facto-StoreConnectionManager.js +441 -0
- package/source/services/Retold-Facto-ThroughputMonitor.js +478 -0
- package/source/services/web-app/codemirror-entry.js +7 -0
- package/source/services/web-app/pict-app/Pict-Application-Facto-Configuration.json +9 -0
- package/source/services/web-app/pict-app/Pict-Application-Facto.js +70 -0
- package/source/services/web-app/pict-app/Pict-Facto-Bundle.js +11 -0
- package/source/services/web-app/pict-app/providers/Pict-Provider-Facto-UI.js +66 -0
- package/source/services/web-app/pict-app/providers/Pict-Provider-Facto.js +69 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Catalog.js +93 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Connections.js +42 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Datasets.js +605 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Projections.js +188 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Scanner.js +80 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Schema.js +116 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Sources.js +104 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Catalog.js +526 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Datasets.js +173 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Ingest.js +259 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Layout.js +191 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Projections.js +231 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Records.js +326 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Scanner.js +624 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Sources.js +201 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Throughput.js +456 -0
- package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full-Configuration.json +14 -0
- package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full.js +391 -0
- package/source/services/web-app/pict-app-full/providers/PictRouter-Facto-Configuration.json +56 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-BottomBar.js +68 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Connections.js +340 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboard.js +149 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboards.js +819 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Datasets.js +178 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-IngestJobs.js +99 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Layout.js +62 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-MappingEditor.js +158 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-ProjectionDetail.js +1120 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Projections.js +172 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-QueryPanel.js +119 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-RecordViewer.js +663 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Records.js +648 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Scanner.js +1017 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDetail.js +1404 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDocEditor.js +1036 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaEditor.js +636 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaResearch.js +357 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceDetail.js +822 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceEditor.js +1036 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceResearch.js +487 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Sources.js +165 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Throughput.js +439 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-TopBar.js +335 -0
- package/source/services/web-app/pict-app-full/views/projections/Facto-Projections-Constants.js +71 -0
- package/source/services/web-app/web/chart.min.js +20 -0
- package/source/services/web-app/web/codemirror-bundle.js +30099 -0
- package/source/services/web-app/web/css/facto-themes.css +467 -0
- package/source/services/web-app/web/css/facto.css +502 -0
- package/source/services/web-app/web/index.html +28 -0
- package/source/services/web-app/web/retold-facto.js +12138 -0
- package/source/services/web-app/web/retold-facto.js.map +1 -0
- package/source/services/web-app/web/retold-facto.min.js +2 -0
- package/source/services/web-app/web/retold-facto.min.js.map +1 -0
- package/source/services/web-app/web/simple/index.html +17 -0
- package/test/Facto_Browser_Integration_tests.js +798 -0
- package/test/RetoldFacto_tests.js +4117 -0
- package/test/fixtures/weather-readings.csv +17 -0
- package/test/fixtures/weather-stations.csv +9 -0
- package/test/model/MeadowModel-Extended.json +8497 -0
- package/test/model/MeadowModel-PICT.json +1 -0
- package/test/model/MeadowModel.json +1355 -0
- package/test/model/ddl/Facto.ddl +225 -0
- package/test/model/fable-configuration.json +14 -0
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
const libPictView = require('pict-view');
|
|
2
|
+
|
|
3
|
+
class FactoSourcesView extends libPictView
|
|
4
|
+
{
|
|
5
|
+
constructor(pFable, pOptions, pServiceHash)
|
|
6
|
+
{
|
|
7
|
+
super(pFable, pOptions, pServiceHash);
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
onAfterRender()
|
|
11
|
+
{
|
|
12
|
+
// Load sources from API on first render
|
|
13
|
+
this.pict.providers.Facto.loadSources().then(
|
|
14
|
+
() =>
|
|
15
|
+
{
|
|
16
|
+
this.refreshList();
|
|
17
|
+
}).catch(
|
|
18
|
+
(pError) =>
|
|
19
|
+
{
|
|
20
|
+
this.pict.views['Pict-Section-Modal'].toast('Error loading sources: ' + pError.message, {type: 'error'});
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
refreshList()
|
|
25
|
+
{
|
|
26
|
+
let tmpContainer = document.getElementById('facto-sources-list');
|
|
27
|
+
if (!tmpContainer) return;
|
|
28
|
+
|
|
29
|
+
let tmpSources = this.pict.AppData.Facto.Sources;
|
|
30
|
+
if (!tmpSources || tmpSources.length === 0)
|
|
31
|
+
{
|
|
32
|
+
tmpContainer.innerHTML = '<p style="color:#888; font-style:italic;">No sources registered yet.</p>';
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
let tmpHtml = '<table><thead><tr><th>ID</th><th>Name</th><th>Type</th><th>URL</th><th>Active</th><th>Actions</th></tr></thead><tbody>';
|
|
37
|
+
for (let i = 0; i < tmpSources.length; i++)
|
|
38
|
+
{
|
|
39
|
+
let tmpSource = tmpSources[i];
|
|
40
|
+
let tmpActiveLabel = tmpSource.Active ? '<span style="color:#28a745;">Active</span>' : '<span style="color:#888;">Inactive</span>';
|
|
41
|
+
let tmpToggleBtn = tmpSource.Active
|
|
42
|
+
? '<button class="secondary" style="padding:4px 8px; font-size:0.8em;" onclick="pict.views[\'Facto-Sources\'].toggleActive(' + tmpSource.IDSource + ', false)">Deactivate</button>'
|
|
43
|
+
: '<button class="success" style="padding:4px 8px; font-size:0.8em;" onclick="pict.views[\'Facto-Sources\'].toggleActive(' + tmpSource.IDSource + ', true)">Activate</button>';
|
|
44
|
+
tmpHtml += '<tr>';
|
|
45
|
+
tmpHtml += '<td>' + (tmpSource.IDSource || '') + '</td>';
|
|
46
|
+
tmpHtml += '<td>' + (tmpSource.Name || '') + '</td>';
|
|
47
|
+
tmpHtml += '<td>' + (tmpSource.Type || '') + '</td>';
|
|
48
|
+
tmpHtml += '<td style="max-width:200px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap;">' + (tmpSource.URL || '') + '</td>';
|
|
49
|
+
tmpHtml += '<td>' + tmpActiveLabel + '</td>';
|
|
50
|
+
tmpHtml += '<td>' + tmpToggleBtn + '</td>';
|
|
51
|
+
tmpHtml += '</tr>';
|
|
52
|
+
}
|
|
53
|
+
tmpHtml += '</tbody></table>';
|
|
54
|
+
tmpContainer.innerHTML = tmpHtml;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
toggleActive(pIDSource, pActivate)
|
|
58
|
+
{
|
|
59
|
+
let tmpPromise = pActivate
|
|
60
|
+
? this.pict.providers.Facto.activateSource(pIDSource)
|
|
61
|
+
: this.pict.providers.Facto.deactivateSource(pIDSource);
|
|
62
|
+
|
|
63
|
+
tmpPromise.then(
|
|
64
|
+
() =>
|
|
65
|
+
{
|
|
66
|
+
return this.pict.providers.Facto.loadSources();
|
|
67
|
+
}).then(
|
|
68
|
+
() =>
|
|
69
|
+
{
|
|
70
|
+
this.refreshList();
|
|
71
|
+
}).catch(
|
|
72
|
+
(pError) =>
|
|
73
|
+
{
|
|
74
|
+
this.pict.views['Pict-Section-Modal'].toast('Error: ' + pError.message, {type: 'error'});
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
addSource()
|
|
79
|
+
{
|
|
80
|
+
let tmpName = this.pict.providers.FactoUI.getVal('facto-source-name');
|
|
81
|
+
let tmpType = this.pict.providers.FactoUI.getVal('facto-source-type');
|
|
82
|
+
let tmpURL = this.pict.providers.FactoUI.getVal('facto-source-url');
|
|
83
|
+
let tmpProtocol = this.pict.providers.FactoUI.getVal('facto-source-protocol');
|
|
84
|
+
|
|
85
|
+
if (!tmpName)
|
|
86
|
+
{
|
|
87
|
+
this.pict.views['Pict-Section-Modal'].toast('Name is required', {type: 'warning'});
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
this.pict.providers.Facto.createSource(
|
|
92
|
+
{
|
|
93
|
+
Name: tmpName,
|
|
94
|
+
Type: tmpType,
|
|
95
|
+
URL: tmpURL,
|
|
96
|
+
Protocol: tmpProtocol,
|
|
97
|
+
Active: 1
|
|
98
|
+
}).then(
|
|
99
|
+
(pResponse) =>
|
|
100
|
+
{
|
|
101
|
+
if (pResponse && pResponse.IDSource)
|
|
102
|
+
{
|
|
103
|
+
this.pict.views['Pict-Section-Modal'].toast('Source created: ' + pResponse.Name, {type: 'success'});
|
|
104
|
+
// Clear form
|
|
105
|
+
if (document.getElementById('facto-source-name')) document.getElementById('facto-source-name').value = '';
|
|
106
|
+
if (document.getElementById('facto-source-url')) document.getElementById('facto-source-url').value = '';
|
|
107
|
+
// Reload list
|
|
108
|
+
return this.pict.providers.Facto.loadSources();
|
|
109
|
+
}
|
|
110
|
+
else
|
|
111
|
+
{
|
|
112
|
+
this.pict.views['Pict-Section-Modal'].toast('Error creating source', {type: 'error'});
|
|
113
|
+
}
|
|
114
|
+
}).then(
|
|
115
|
+
() =>
|
|
116
|
+
{
|
|
117
|
+
this.refreshList();
|
|
118
|
+
}).catch(
|
|
119
|
+
(pError) =>
|
|
120
|
+
{
|
|
121
|
+
this.pict.views['Pict-Section-Modal'].toast('Error: ' + pError.message, {type: 'error'});
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
module.exports = FactoSourcesView;
|
|
127
|
+
|
|
128
|
+
module.exports.default_configuration =
|
|
129
|
+
{
|
|
130
|
+
ViewIdentifier: 'Facto-Sources',
|
|
131
|
+
DefaultRenderable: 'Facto-Sources',
|
|
132
|
+
DefaultDestinationAddress: '#Facto-Section-Sources',
|
|
133
|
+
Templates:
|
|
134
|
+
[
|
|
135
|
+
{
|
|
136
|
+
Hash: 'Facto-Sources',
|
|
137
|
+
Template: /*html*/`
|
|
138
|
+
<div class="accordion-row">
|
|
139
|
+
<div class="accordion-number">1</div>
|
|
140
|
+
<div class="accordion-card open" id="facto-card-sources">
|
|
141
|
+
<div class="accordion-header" onclick="pict.views['Facto-Layout'].toggleSection('facto-card-sources')">
|
|
142
|
+
<span class="accordion-title">Sources</span>
|
|
143
|
+
<span class="accordion-preview">Manage data sources</span>
|
|
144
|
+
<span class="accordion-toggle">▼</span>
|
|
145
|
+
</div>
|
|
146
|
+
<div class="accordion-body">
|
|
147
|
+
<p style="margin-bottom:12px; color:#666; font-size:0.9em;">Data sources describe where ingested data originates -- websites, APIs, FTP servers, OCR results, ML outputs, etc.</p>
|
|
148
|
+
<div id="facto-sources-list"></div>
|
|
149
|
+
|
|
150
|
+
<h3 style="margin-top:16px; margin-bottom:8px; font-size:1em; color:#444;">Add Source</h3>
|
|
151
|
+
<div class="inline-group">
|
|
152
|
+
<div>
|
|
153
|
+
<label for="facto-source-name">Name</label>
|
|
154
|
+
<input type="text" id="facto-source-name" placeholder="e.g. US Census Bureau API">
|
|
155
|
+
</div>
|
|
156
|
+
<div>
|
|
157
|
+
<label for="facto-source-type">Type</label>
|
|
158
|
+
<select id="facto-source-type">
|
|
159
|
+
<option value="API">API</option>
|
|
160
|
+
<option value="File">File</option>
|
|
161
|
+
<option value="FTP">FTP</option>
|
|
162
|
+
<option value="Web">Web</option>
|
|
163
|
+
<option value="OCR">OCR</option>
|
|
164
|
+
<option value="ML">ML</option>
|
|
165
|
+
<option value="Manual">Manual</option>
|
|
166
|
+
</select>
|
|
167
|
+
</div>
|
|
168
|
+
</div>
|
|
169
|
+
<div class="inline-group">
|
|
170
|
+
<div>
|
|
171
|
+
<label for="facto-source-url">URL</label>
|
|
172
|
+
<input type="text" id="facto-source-url" placeholder="https://api.example.gov/data">
|
|
173
|
+
</div>
|
|
174
|
+
<div>
|
|
175
|
+
<label for="facto-source-protocol">Protocol</label>
|
|
176
|
+
<select id="facto-source-protocol">
|
|
177
|
+
<option value="HTTPS">HTTPS</option>
|
|
178
|
+
<option value="HTTP">HTTP</option>
|
|
179
|
+
<option value="FTP">FTP</option>
|
|
180
|
+
<option value="SFTP">SFTP</option>
|
|
181
|
+
<option value="Local">Local</option>
|
|
182
|
+
</select>
|
|
183
|
+
</div>
|
|
184
|
+
</div>
|
|
185
|
+
<button class="primary" onclick="pict.views['Facto-Sources'].addSource()">Add Source</button>
|
|
186
|
+
|
|
187
|
+
</div>
|
|
188
|
+
</div>
|
|
189
|
+
</div>
|
|
190
|
+
`
|
|
191
|
+
}
|
|
192
|
+
],
|
|
193
|
+
Renderables:
|
|
194
|
+
[
|
|
195
|
+
{
|
|
196
|
+
RenderableHash: 'Facto-Sources',
|
|
197
|
+
TemplateHash: 'Facto-Sources',
|
|
198
|
+
DestinationAddress: '#Facto-Section-Sources'
|
|
199
|
+
}
|
|
200
|
+
]
|
|
201
|
+
};
|
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PictView-Facto-Throughput
|
|
3
|
+
*
|
|
4
|
+
* Live temporal histogram showing pipeline throughput across three stages:
|
|
5
|
+
* - Extracted (records parsed from source files)
|
|
6
|
+
* - Transformed (records mapped via TabularTransform)
|
|
7
|
+
* - Written (records persisted to Facto store)
|
|
8
|
+
*
|
|
9
|
+
* Renders four histograms:
|
|
10
|
+
* 1. Extracted — blue
|
|
11
|
+
* 2. Transformed — amber
|
|
12
|
+
* 3. Written — green
|
|
13
|
+
* 4. Combined (stacked) — all three overlaid
|
|
14
|
+
*
|
|
15
|
+
* Polls /facto/throughput every 500ms while active.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const libPictView = require('pict-view');
|
|
19
|
+
|
|
20
|
+
const STAGE_COLORS =
|
|
21
|
+
{
|
|
22
|
+
extracted: { bar: '#4a90d9', bg: 'rgba(74,144,217,0.15)', label: 'Extracted' },
|
|
23
|
+
transformed: { bar: '#d09818', bg: 'rgba(208,152,24,0.15)', label: 'Transformed' },
|
|
24
|
+
written: { bar: '#3a9468', bg: 'rgba(58,148,104,0.15)', label: 'Written' },
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const HISTOGRAM_HEIGHT = 80;
|
|
28
|
+
const BAR_WIDTH = 6;
|
|
29
|
+
const BAR_GAP = 2;
|
|
30
|
+
const POLL_INTERVAL_MS = 500;
|
|
31
|
+
|
|
32
|
+
class FactoThroughputView extends libPictView
|
|
33
|
+
{
|
|
34
|
+
constructor(pFable, pOptions, pServiceHash)
|
|
35
|
+
{
|
|
36
|
+
super(pFable, pOptions, pServiceHash);
|
|
37
|
+
|
|
38
|
+
this._pollTimer = null;
|
|
39
|
+
this._isPolling = false;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
onAfterRender()
|
|
43
|
+
{
|
|
44
|
+
// Don't auto-start polling — user clicks "Start Monitoring"
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
startMonitoring()
|
|
48
|
+
{
|
|
49
|
+
if (this._isPolling) return;
|
|
50
|
+
this._isPolling = true;
|
|
51
|
+
|
|
52
|
+
let tmpBtn = document.getElementById('facto-throughput-toggle');
|
|
53
|
+
if (tmpBtn)
|
|
54
|
+
{
|
|
55
|
+
tmpBtn.textContent = 'Stop Monitoring';
|
|
56
|
+
tmpBtn.className = 'danger';
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
this._poll();
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
stopMonitoring()
|
|
63
|
+
{
|
|
64
|
+
this._isPolling = false;
|
|
65
|
+
if (this._pollTimer)
|
|
66
|
+
{
|
|
67
|
+
clearTimeout(this._pollTimer);
|
|
68
|
+
this._pollTimer = null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
let tmpBtn = document.getElementById('facto-throughput-toggle');
|
|
72
|
+
if (tmpBtn)
|
|
73
|
+
{
|
|
74
|
+
tmpBtn.textContent = 'Start Monitoring';
|
|
75
|
+
tmpBtn.className = 'primary';
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
toggleMonitoring()
|
|
80
|
+
{
|
|
81
|
+
if (this._isPolling)
|
|
82
|
+
{
|
|
83
|
+
this.stopMonitoring();
|
|
84
|
+
}
|
|
85
|
+
else
|
|
86
|
+
{
|
|
87
|
+
this.startMonitoring();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
_poll()
|
|
92
|
+
{
|
|
93
|
+
if (!this._isPolling) return;
|
|
94
|
+
|
|
95
|
+
fetch('/facto/throughput?duration=30')
|
|
96
|
+
.then((pResponse) => pResponse.json())
|
|
97
|
+
.then((pData) =>
|
|
98
|
+
{
|
|
99
|
+
this._renderHistograms(pData);
|
|
100
|
+
this._pollTimer = setTimeout(() => this._poll(), POLL_INTERVAL_MS);
|
|
101
|
+
})
|
|
102
|
+
.catch((pError) =>
|
|
103
|
+
{
|
|
104
|
+
this._pollTimer = setTimeout(() => this._poll(), POLL_INTERVAL_MS * 4);
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
_renderHistograms(pData)
|
|
109
|
+
{
|
|
110
|
+
let tmpBuckets = pData.buckets || [];
|
|
111
|
+
let tmpMaxValue = pData.maxValue || 1;
|
|
112
|
+
let tmpTotals = { extracted: 0, transformed: 0, written: 0 };
|
|
113
|
+
|
|
114
|
+
for (let i = 0; i < tmpBuckets.length; i++)
|
|
115
|
+
{
|
|
116
|
+
tmpTotals.extracted += tmpBuckets[i].extracted;
|
|
117
|
+
tmpTotals.transformed += tmpBuckets[i].transformed;
|
|
118
|
+
tmpTotals.written += tmpBuckets[i].written;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Find max for stacked chart
|
|
122
|
+
let tmpStackedMax = 1;
|
|
123
|
+
for (let i = 0; i < tmpBuckets.length; i++)
|
|
124
|
+
{
|
|
125
|
+
let tmpTotal = tmpBuckets[i].extracted + tmpBuckets[i].transformed + tmpBuckets[i].written;
|
|
126
|
+
if (tmpTotal > tmpStackedMax) tmpStackedMax = tmpTotal;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
let tmpContainer = document.getElementById('facto-throughput-charts');
|
|
130
|
+
if (!tmpContainer) return;
|
|
131
|
+
|
|
132
|
+
let tmpHtml = '';
|
|
133
|
+
|
|
134
|
+
// Summary totals
|
|
135
|
+
tmpHtml += '<div style="display:flex; gap:24px; margin-bottom:12px; flex-wrap:wrap;">';
|
|
136
|
+
tmpHtml += this._renderTotalBadge('Extracted', tmpTotals.extracted, STAGE_COLORS.extracted.bar);
|
|
137
|
+
tmpHtml += this._renderTotalBadge('Transformed', tmpTotals.transformed, STAGE_COLORS.transformed.bar);
|
|
138
|
+
tmpHtml += this._renderTotalBadge('Written', tmpTotals.written, STAGE_COLORS.written.bar);
|
|
139
|
+
|
|
140
|
+
// Active run indicator
|
|
141
|
+
if (pData.activeRun)
|
|
142
|
+
{
|
|
143
|
+
let tmpElapsed = ((Date.now() - pData.activeRun.startTime) / 1000).toFixed(1);
|
|
144
|
+
tmpHtml += '<div style="font-size:0.85em; color:var(--facto-text-secondary); display:flex; align-items:center; gap:6px;">';
|
|
145
|
+
tmpHtml += '<span style="display:inline-block; width:8px; height:8px; border-radius:50%; background:#3a9468; animation:pulse 1s infinite;"></span>';
|
|
146
|
+
tmpHtml += pData.activeRun.label + ' (' + tmpElapsed + 's)';
|
|
147
|
+
tmpHtml += '</div>';
|
|
148
|
+
}
|
|
149
|
+
tmpHtml += '</div>';
|
|
150
|
+
|
|
151
|
+
// Individual histograms
|
|
152
|
+
tmpHtml += '<div style="display:grid; grid-template-columns:1fr 1fr; gap:12px; margin-bottom:12px;">';
|
|
153
|
+
tmpHtml += this._renderSingleHistogram('Extracted', 'extracted', tmpBuckets, tmpMaxValue);
|
|
154
|
+
tmpHtml += this._renderSingleHistogram('Transformed', 'transformed', tmpBuckets, tmpMaxValue);
|
|
155
|
+
tmpHtml += this._renderSingleHistogram('Written', 'written', tmpBuckets, tmpMaxValue);
|
|
156
|
+
tmpHtml += this._renderStackedHistogram('Combined', tmpBuckets, tmpStackedMax);
|
|
157
|
+
tmpHtml += '</div>';
|
|
158
|
+
|
|
159
|
+
tmpContainer.innerHTML = tmpHtml;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
_renderTotalBadge(pLabel, pCount, pColor)
|
|
163
|
+
{
|
|
164
|
+
return '<div style="display:flex; align-items:center; gap:6px;">'
|
|
165
|
+
+ '<div style="width:12px; height:12px; border-radius:2px; background:' + pColor + ';"></div>'
|
|
166
|
+
+ '<span style="font-weight:600; font-size:0.9em;">' + pLabel + ':</span>'
|
|
167
|
+
+ '<span style="font-size:0.9em; color:var(--facto-text-secondary);">' + pCount.toLocaleString() + '</span>'
|
|
168
|
+
+ '</div>';
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
_renderSingleHistogram(pTitle, pStage, pBuckets, pMaxValue)
|
|
172
|
+
{
|
|
173
|
+
let tmpColor = STAGE_COLORS[pStage];
|
|
174
|
+
let tmpBarsHtml = '';
|
|
175
|
+
let tmpMax = Math.max(pMaxValue, 1);
|
|
176
|
+
|
|
177
|
+
for (let i = 0; i < pBuckets.length; i++)
|
|
178
|
+
{
|
|
179
|
+
let tmpVal = pBuckets[i][pStage];
|
|
180
|
+
let tmpPct = (tmpVal / tmpMax) * 100;
|
|
181
|
+
let tmpBarColor = tmpVal > 0 ? tmpColor.bar : 'transparent';
|
|
182
|
+
|
|
183
|
+
tmpBarsHtml += '<div style="'
|
|
184
|
+
+ 'display:inline-block;'
|
|
185
|
+
+ 'width:' + BAR_WIDTH + 'px;'
|
|
186
|
+
+ 'height:' + HISTOGRAM_HEIGHT + 'px;'
|
|
187
|
+
+ 'margin-right:' + BAR_GAP + 'px;'
|
|
188
|
+
+ 'position:relative;'
|
|
189
|
+
+ 'vertical-align:bottom;'
|
|
190
|
+
+ '">'
|
|
191
|
+
+ '<div style="'
|
|
192
|
+
+ 'position:absolute; bottom:0; left:0; right:0;'
|
|
193
|
+
+ 'height:' + tmpPct + '%;'
|
|
194
|
+
+ 'background:' + tmpBarColor + ';'
|
|
195
|
+
+ 'border-radius:1px 1px 0 0;'
|
|
196
|
+
+ 'transition:height 0.3s;'
|
|
197
|
+
+ '" title="' + tmpVal + ' records"></div>'
|
|
198
|
+
+ '</div>';
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
return '<div style="background:' + tmpColor.bg + '; border-radius:6px; padding:10px;">'
|
|
202
|
+
+ '<div style="font-size:0.8em; font-weight:600; color:' + tmpColor.bar + '; margin-bottom:6px;">' + pTitle + '</div>'
|
|
203
|
+
+ '<div style="overflow-x:auto; white-space:nowrap; display:flex; align-items:flex-end; height:' + HISTOGRAM_HEIGHT + 'px; border-bottom:1px solid rgba(0,0,0,0.1);">'
|
|
204
|
+
+ tmpBarsHtml
|
|
205
|
+
+ '</div>'
|
|
206
|
+
+ '</div>';
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// ─────────────────────────────────────────────
|
|
210
|
+
// Historical run browsing
|
|
211
|
+
// ─────────────────────────────────────────────
|
|
212
|
+
|
|
213
|
+
loadRunHistory()
|
|
214
|
+
{
|
|
215
|
+
fetch('/facto/throughput/runs?limit=10')
|
|
216
|
+
.then((pResponse) => pResponse.json())
|
|
217
|
+
.then((pRuns) =>
|
|
218
|
+
{
|
|
219
|
+
this._renderRunHistory(pRuns);
|
|
220
|
+
})
|
|
221
|
+
.catch(() => {});
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
loadHistoricalRun(pLabel)
|
|
225
|
+
{
|
|
226
|
+
this._activeHistoricalRun = pLabel;
|
|
227
|
+
this._activeDatasetFilter = null;
|
|
228
|
+
|
|
229
|
+
fetch('/facto/throughput/run/' + encodeURIComponent(pLabel))
|
|
230
|
+
.then((pResponse) => pResponse.json())
|
|
231
|
+
.then((pData) =>
|
|
232
|
+
{
|
|
233
|
+
pData.historicalRun = pLabel;
|
|
234
|
+
this._renderHistograms(pData);
|
|
235
|
+
this._loadDatasetBreakdown(pLabel);
|
|
236
|
+
})
|
|
237
|
+
.catch(() => {});
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
filterByDataset(pDataset)
|
|
241
|
+
{
|
|
242
|
+
if (!this._activeHistoricalRun) return;
|
|
243
|
+
this._activeDatasetFilter = pDataset;
|
|
244
|
+
|
|
245
|
+
let tmpUrl = '/facto/throughput/run/' + encodeURIComponent(this._activeHistoricalRun);
|
|
246
|
+
if (pDataset)
|
|
247
|
+
{
|
|
248
|
+
tmpUrl += '?dataset=' + encodeURIComponent(pDataset);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
fetch(tmpUrl)
|
|
252
|
+
.then((pResponse) => pResponse.json())
|
|
253
|
+
.then((pData) =>
|
|
254
|
+
{
|
|
255
|
+
pData.historicalRun = this._activeHistoricalRun;
|
|
256
|
+
pData.datasetFilter = pDataset;
|
|
257
|
+
this._renderHistograms(pData);
|
|
258
|
+
})
|
|
259
|
+
.catch(() => {});
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
_loadDatasetBreakdown(pLabel)
|
|
263
|
+
{
|
|
264
|
+
fetch('/facto/throughput/run/' + encodeURIComponent(pLabel) + '/datasets')
|
|
265
|
+
.then((pResponse) => pResponse.json())
|
|
266
|
+
.then((pDatasets) =>
|
|
267
|
+
{
|
|
268
|
+
this._renderDatasetBreakdown(pDatasets);
|
|
269
|
+
})
|
|
270
|
+
.catch(() => {});
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
_renderRunHistory(pRuns)
|
|
274
|
+
{
|
|
275
|
+
let tmpContainer = document.getElementById('facto-throughput-history');
|
|
276
|
+
if (!tmpContainer) return;
|
|
277
|
+
|
|
278
|
+
if (!pRuns || pRuns.length === 0)
|
|
279
|
+
{
|
|
280
|
+
tmpContainer.innerHTML = '<p style="color:#aaa; font-size:0.85em; font-style:italic;">No historical runs found.</p>';
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
let tmpHtml = '<div style="font-size:0.85em; font-weight:600; color:var(--facto-text-heading); margin-bottom:6px;">Run History</div>';
|
|
285
|
+
tmpHtml += '<div style="display:flex; flex-wrap:wrap; gap:6px;">';
|
|
286
|
+
for (let i = 0; i < pRuns.length; i++)
|
|
287
|
+
{
|
|
288
|
+
let tmpRun = pRuns[i];
|
|
289
|
+
let tmpDate = new Date(tmpRun.startTime).toLocaleString();
|
|
290
|
+
let tmpDatasets = tmpRun.datasets.length;
|
|
291
|
+
tmpHtml += '<button class="secondary" style="padding:4px 10px; font-size:0.8em;" '
|
|
292
|
+
+ 'onclick="pict.views[\'Facto-Throughput\'].loadHistoricalRun(\'' + tmpRun.label.replace(/'/g, "\\'") + '\')" '
|
|
293
|
+
+ 'title="' + tmpDate + ' — ' + tmpRun.eventCount + ' events, ' + tmpDatasets + ' datasets">'
|
|
294
|
+
+ tmpRun.label.substring(0, 30) + (tmpRun.label.length > 30 ? '...' : '')
|
|
295
|
+
+ '</button>';
|
|
296
|
+
}
|
|
297
|
+
tmpHtml += '</div>';
|
|
298
|
+
|
|
299
|
+
tmpContainer.innerHTML = tmpHtml;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
_renderDatasetBreakdown(pDatasets)
|
|
303
|
+
{
|
|
304
|
+
let tmpContainer = document.getElementById('facto-throughput-datasets');
|
|
305
|
+
if (!tmpContainer) return;
|
|
306
|
+
|
|
307
|
+
if (!pDatasets || pDatasets.length === 0)
|
|
308
|
+
{
|
|
309
|
+
tmpContainer.innerHTML = '';
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// Find max total for scaling
|
|
314
|
+
let tmpMaxTotal = 1;
|
|
315
|
+
for (let i = 0; i < pDatasets.length; i++)
|
|
316
|
+
{
|
|
317
|
+
if (pDatasets[i].total > tmpMaxTotal) tmpMaxTotal = pDatasets[i].total;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
let tmpHtml = '<div style="font-size:0.85em; font-weight:600; color:var(--facto-text-heading); margin-bottom:8px;">Per-Dataset Breakdown</div>';
|
|
321
|
+
tmpHtml += '<div style="font-size:0.8em; margin-bottom:4px; color:var(--facto-text-tertiary);">Click a dataset to filter the histogram:</div>';
|
|
322
|
+
|
|
323
|
+
for (let i = 0; i < pDatasets.length; i++)
|
|
324
|
+
{
|
|
325
|
+
let tmpDS = pDatasets[i];
|
|
326
|
+
let tmpPct = (tmpDS.total / tmpMaxTotal) * 100;
|
|
327
|
+
let tmpActive = this._activeDatasetFilter === tmpDS.dataset;
|
|
328
|
+
|
|
329
|
+
tmpHtml += '<div style="display:flex; align-items:center; gap:8px; margin-bottom:4px; cursor:pointer; '
|
|
330
|
+
+ (tmpActive ? 'background:var(--facto-brand-a15); border-radius:4px; padding:2px 6px;' : 'padding:2px 6px;')
|
|
331
|
+
+ '" onclick="pict.views[\'Facto-Throughput\'].filterByDataset(' + (tmpActive ? 'null' : '\'' + tmpDS.dataset.replace(/'/g, "\\'") + '\'') + ')">';
|
|
332
|
+
tmpHtml += '<span style="min-width:200px; font-family:monospace; font-size:0.9em; color:var(--facto-text);">' + tmpDS.dataset + '</span>';
|
|
333
|
+
|
|
334
|
+
// Stacked bar
|
|
335
|
+
let tmpEPct = (tmpDS.extracted / tmpMaxTotal) * 100;
|
|
336
|
+
let tmpTPct = (tmpDS.transformed / tmpMaxTotal) * 100;
|
|
337
|
+
let tmpWPct = (tmpDS.written / tmpMaxTotal) * 100;
|
|
338
|
+
|
|
339
|
+
tmpHtml += '<div style="flex:1; height:16px; display:flex; border-radius:3px; overflow:hidden; background:var(--facto-bg-elevated);">';
|
|
340
|
+
if (tmpDS.extracted > 0) tmpHtml += '<div style="width:' + tmpEPct + '%; background:' + STAGE_COLORS.extracted.bar + ';" title="Extracted: ' + tmpDS.extracted + '"></div>';
|
|
341
|
+
if (tmpDS.transformed > 0) tmpHtml += '<div style="width:' + tmpTPct + '%; background:' + STAGE_COLORS.transformed.bar + ';" title="Transformed: ' + tmpDS.transformed + '"></div>';
|
|
342
|
+
if (tmpDS.written > 0) tmpHtml += '<div style="width:' + tmpWPct + '%; background:' + STAGE_COLORS.written.bar + ';" title="Written: ' + tmpDS.written + '"></div>';
|
|
343
|
+
tmpHtml += '</div>';
|
|
344
|
+
|
|
345
|
+
tmpHtml += '<span style="min-width:60px; text-align:right; font-size:0.85em; color:var(--facto-text-secondary);">' + tmpDS.total.toLocaleString() + '</span>';
|
|
346
|
+
tmpHtml += '</div>';
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
tmpContainer.innerHTML = tmpHtml;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
// ─────────────────────────────────────────────
|
|
353
|
+
// Histogram rendering
|
|
354
|
+
// ─────────────────────────────────────────────
|
|
355
|
+
|
|
356
|
+
_renderStackedHistogram(pTitle, pBuckets, pMaxValue)
|
|
357
|
+
{
|
|
358
|
+
let tmpBarsHtml = '';
|
|
359
|
+
let tmpMax = Math.max(pMaxValue, 1);
|
|
360
|
+
|
|
361
|
+
for (let i = 0; i < pBuckets.length; i++)
|
|
362
|
+
{
|
|
363
|
+
let tmpE = pBuckets[i].extracted;
|
|
364
|
+
let tmpT = pBuckets[i].transformed;
|
|
365
|
+
let tmpW = pBuckets[i].written;
|
|
366
|
+
|
|
367
|
+
let tmpEPct = (tmpE / tmpMax) * 100;
|
|
368
|
+
let tmpTPct = (tmpT / tmpMax) * 100;
|
|
369
|
+
let tmpWPct = (tmpW / tmpMax) * 100;
|
|
370
|
+
|
|
371
|
+
tmpBarsHtml += '<div style="'
|
|
372
|
+
+ 'display:inline-flex; flex-direction:column-reverse;'
|
|
373
|
+
+ 'width:' + BAR_WIDTH + 'px;'
|
|
374
|
+
+ 'height:' + HISTOGRAM_HEIGHT + 'px;'
|
|
375
|
+
+ 'margin-right:' + BAR_GAP + 'px;'
|
|
376
|
+
+ 'vertical-align:bottom;'
|
|
377
|
+
+ 'align-items:stretch;'
|
|
378
|
+
+ '">';
|
|
379
|
+
|
|
380
|
+
// Stacked bottom to top: written (green), transformed (amber), extracted (blue)
|
|
381
|
+
if (tmpW > 0)
|
|
382
|
+
{
|
|
383
|
+
tmpBarsHtml += '<div style="height:' + tmpWPct + '%; background:' + STAGE_COLORS.written.bar + '; transition:height 0.3s;" title="Written: ' + tmpW + '"></div>';
|
|
384
|
+
}
|
|
385
|
+
if (tmpT > 0)
|
|
386
|
+
{
|
|
387
|
+
tmpBarsHtml += '<div style="height:' + tmpTPct + '%; background:' + STAGE_COLORS.transformed.bar + '; transition:height 0.3s;" title="Transformed: ' + tmpT + '"></div>';
|
|
388
|
+
}
|
|
389
|
+
if (tmpE > 0)
|
|
390
|
+
{
|
|
391
|
+
tmpBarsHtml += '<div style="height:' + tmpEPct + '%; background:' + STAGE_COLORS.extracted.bar + '; transition:height 0.3s;" title="Extracted: ' + tmpE + '"></div>';
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
tmpBarsHtml += '</div>';
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
return '<div style="background:rgba(0,0,0,0.03); border:1px solid var(--facto-border-subtle); border-radius:6px; padding:10px;">'
|
|
398
|
+
+ '<div style="font-size:0.8em; font-weight:600; color:var(--facto-text-heading); margin-bottom:6px;">' + pTitle + ' (stacked)</div>'
|
|
399
|
+
+ '<div style="overflow-x:auto; white-space:nowrap; display:flex; align-items:flex-end; height:' + HISTOGRAM_HEIGHT + 'px; border-bottom:1px solid rgba(0,0,0,0.1);">'
|
|
400
|
+
+ tmpBarsHtml
|
|
401
|
+
+ '</div>'
|
|
402
|
+
+ '</div>';
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
module.exports = FactoThroughputView;
|
|
407
|
+
|
|
408
|
+
module.exports.default_configuration =
|
|
409
|
+
{
|
|
410
|
+
ViewIdentifier: 'Facto-Throughput',
|
|
411
|
+
DefaultRenderable: 'Facto-Throughput',
|
|
412
|
+
DefaultDestinationAddress: '#Facto-Section-Throughput',
|
|
413
|
+
Templates:
|
|
414
|
+
[
|
|
415
|
+
{
|
|
416
|
+
Hash: 'Facto-Throughput',
|
|
417
|
+
Template: /*html*/`
|
|
418
|
+
<div class="accordion-row">
|
|
419
|
+
<div class="accordion-number" style="color:var(--facto-brand);">⚡</div>
|
|
420
|
+
<div class="accordion-card open">
|
|
421
|
+
<div class="accordion-header" onclick="pict.views['Facto-Layout'].toggleSection('facto-card-throughput')" id="facto-card-throughput-header">
|
|
422
|
+
<span class="accordion-title">Pipeline Throughput</span>
|
|
423
|
+
<span class="accordion-preview">Live pipeline throughput monitoring</span>
|
|
424
|
+
<span class="accordion-toggle">▼</span>
|
|
425
|
+
</div>
|
|
426
|
+
<div class="accordion-body" id="facto-card-throughput">
|
|
427
|
+
<p style="margin-bottom:12px; color:#666; font-size:0.9em;">
|
|
428
|
+
Temporal histograms showing record flow through extraction, transformation, and storage stages.
|
|
429
|
+
</p>
|
|
430
|
+
<div style="display:flex; gap:8px; margin-bottom:12px; flex-wrap:wrap;">
|
|
431
|
+
<button id="facto-throughput-toggle" class="primary" onclick="pict.views['Facto-Throughput'].toggleMonitoring()">Start Live Monitoring</button>
|
|
432
|
+
<button class="secondary" onclick="pict.views['Facto-Throughput'].loadRunHistory()">Browse Run History</button>
|
|
433
|
+
</div>
|
|
434
|
+
<div id="facto-throughput-history" style="margin-bottom:12px;"></div>
|
|
435
|
+
<div id="facto-throughput-charts">
|
|
436
|
+
<p style="color:#aaa; font-style:italic;">Click "Start Live Monitoring" for real-time view, or "Browse Run History" to inspect past runs.</p>
|
|
437
|
+
</div>
|
|
438
|
+
<div id="facto-throughput-datasets" style="margin-top:12px;"></div>
|
|
439
|
+
<style>
|
|
440
|
+
@keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.4; } }
|
|
441
|
+
</style>
|
|
442
|
+
</div>
|
|
443
|
+
</div>
|
|
444
|
+
</div>
|
|
445
|
+
`
|
|
446
|
+
}
|
|
447
|
+
],
|
|
448
|
+
Renderables:
|
|
449
|
+
[
|
|
450
|
+
{
|
|
451
|
+
RenderableHash: 'Facto-Throughput',
|
|
452
|
+
TemplateHash: 'Facto-Throughput',
|
|
453
|
+
DestinationAddress: '#Facto-Section-Throughput'
|
|
454
|
+
}
|
|
455
|
+
]
|
|
456
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"Name": "Retold Facto",
|
|
3
|
+
"Hash": "Facto-Full",
|
|
4
|
+
"MainViewportViewIdentifier": "Facto-Full-Layout",
|
|
5
|
+
"MainViewportDestinationAddress": "#Facto-Full-Application-Container",
|
|
6
|
+
"MainViewportDefaultDataAddress": "AppData.Facto",
|
|
7
|
+
"AutoSolveAfterInitialize": true,
|
|
8
|
+
"AutoRenderMainViewportViewAfterInitialize": false,
|
|
9
|
+
"AutoRenderViewsAfterInitialize": false,
|
|
10
|
+
"pict_configuration":
|
|
11
|
+
{
|
|
12
|
+
"Product": "Retold-Facto-Full"
|
|
13
|
+
}
|
|
14
|
+
}
|