sqlite-hub 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -13
- package/database.sqlite +0 -0
- package/fill.js +526 -0
- package/frontend/js/api.js +70 -11
- package/frontend/js/app.js +104 -32
- package/frontend/js/components/emptyState.js +42 -46
- package/frontend/js/components/modal.js +95 -0
- package/frontend/js/components/queryEditor.js +3 -2
- package/frontend/js/components/rowEditorPanel.js +145 -6
- package/frontend/js/components/sidebar.js +5 -5
- package/frontend/js/store.js +303 -34
- package/frontend/js/views/data.js +94 -70
- package/frontend/js/views/editor.js +2 -0
- package/frontend/js/views/settings.js +1 -1
- package/frontend/styles/tailwind.generated.css +1 -1
- package/frontend/styles/tokens.css +95 -95
- package/package.json +1 -1
- package/server/routes/data.js +3 -0
- package/server/routes/export.js +97 -12
- package/server/services/sqlite/dataBrowserService.js +25 -4
- package/server/services/sqlite/exportService.js +74 -15
- package/server/services/sqlite/introspection.js +199 -1
- package/server/services/sqlite/sqlExecutor.js +2 -0
- package/server/services/sqlite/tableFilter.js +75 -0
- package/server/utils/csv.js +30 -4
- package/tests/check-constraint-options.test.js +76 -0
- package/tests/sql-identifier-safety.test.js +59 -0
package/README.md
CHANGED
|
@@ -13,11 +13,14 @@ Many database tools are powerful, but feel oversized when all you want is to ins
|
|
|
13
13
|
SQLite Hub keeps that workflow sharp:
|
|
14
14
|
|
|
15
15
|
- browse tables and rows
|
|
16
|
-
-
|
|
17
|
-
-
|
|
18
|
-
-
|
|
16
|
+
- filter, sort, page through, and export table data
|
|
17
|
+
- inspect schema, structure, and relationships
|
|
18
|
+
- edit records in place with an SQL diff preview before saving
|
|
19
|
+
- export tables and query results as CSV, TSV, Markdown, duplicate as table
|
|
19
20
|
- create simple local backups of the active database
|
|
20
|
-
- run SQL in a syntax-highlighted editor
|
|
21
|
+
- run SQL in a syntax-highlighted editor with history, messages, and performance metrics
|
|
22
|
+
- turn query-history results into local charts
|
|
23
|
+
- create and edit tables with a live SQL preview
|
|
21
24
|
- stay local and move fast
|
|
22
25
|
|
|
23
26
|
## Features
|
|
@@ -26,25 +29,53 @@ SQLite Hub keeps that workflow sharp:
|
|
|
26
29
|
|
|
27
30
|

|
|
28
31
|
|
|
29
|
-
Inspect tables, columns, types, and schema details without losing pace.
|
|
32
|
+
Inspect tables, columns, types, indexes, foreign keys, and schema details without losing pace. The graph view visualizes relationships, the table list is searchable, and SQLite Hub remembers the last selected table while you move between views.
|
|
30
33
|
|
|
31
34
|
### Data browser
|
|
32
35
|
|
|
33
36
|

|
|
34
37
|
|
|
35
|
-
Scan rows, sort fast, move through local data quickly, and export full tables as CSV.
|
|
38
|
+
Scan rows, sort fast, move through local data quickly, and export full tables as CSV. The Data browser supports table search, page sizes up to 250 rows, and advanced filters with column/operator/value controls. Text filters support case-insensitive `contains`, `not contains`, and exact `equals` matching.
|
|
36
39
|
|
|
37
40
|
### Row editing
|
|
38
41
|
|
|
39
42
|

|
|
40
43
|
|
|
41
|
-
Open one record, edit it in place,
|
|
44
|
+
Open one record, edit it in place, preview the SQL diff, then commit. SQLite Hub only enables row edits when it can target a stable row identity safely.
|
|
42
45
|
|
|
43
46
|
### SQL editor
|
|
44
47
|
|
|
45
48
|

|
|
46
49
|
|
|
47
|
-
Write queries in a syntax-highlighted editor, inspect results in the same workflow, and export result sets as CSV.
|
|
50
|
+
Write queries in a syntax-highlighted editor, inspect results in the same workflow, and export result sets as CSV. Query drafts survive reloads, query history can be searched and saved, and direct single-table `SELECT` results can be edited from the result grid.
|
|
51
|
+
|
|
52
|
+
The bottom panel keeps separate tabs for:
|
|
53
|
+
|
|
54
|
+
- Results
|
|
55
|
+
- Performance, including execution time, statement count, returned rows, affected rows, and serialized result memory size
|
|
56
|
+
- Messages, including the executed query and statement updates/errors
|
|
57
|
+
|
|
58
|
+
Potentially destructive statements are tracked in query history, and SQLite Hub keeps the active result tab instead of forcing you back to Results after every execution.
|
|
59
|
+
|
|
60
|
+
### Query history
|
|
61
|
+
|
|
62
|
+
SQLite Hub stores query history per database. You can search SQL, titles, and notes; mark useful queries as saved; re-run previous queries; and execute saved queries from the CLI.
|
|
63
|
+
|
|
64
|
+
### Charts
|
|
65
|
+
|
|
66
|
+
Create charts from chartable `SELECT` query-history entries. Charts can be saved per query, reopened later, and rendered from live query results.
|
|
67
|
+
|
|
68
|
+
### Table Designer
|
|
69
|
+
|
|
70
|
+
Create and edit SQLite tables from the UI. The Table Designer includes a searchable table list, column controls, CSV import drafting, and a live SQL preview that can be hidden or shown.
|
|
71
|
+
|
|
72
|
+
### Media Tagging
|
|
73
|
+
|
|
74
|
+
Configure a media table, tag table, and mapping table, then work through a tagging queue for image, video, and audio assets. The workflow supports preview controls, skipped items, parent tags, and applying selected tags to the current media row.
|
|
75
|
+
|
|
76
|
+
### UI preferences
|
|
77
|
+
|
|
78
|
+
SQLite Hub remembers common workspace preferences in local storage, including hidden panels, selected editor tabs, query drafts, chart panels, table row size, and Table Designer preview visibility.
|
|
48
79
|
|
|
49
80
|
### Simple backups
|
|
50
81
|
|
|
@@ -69,7 +100,7 @@ brew install sqlite-hub
|
|
|
69
100
|
npm install -g sqlite-hub
|
|
70
101
|
```
|
|
71
102
|
|
|
72
|
-
##
|
|
103
|
+
## Alternative port
|
|
73
104
|
|
|
74
105
|
```bash
|
|
75
106
|
sqlite-hub --port:4174
|
|
@@ -132,7 +163,7 @@ sqlite-hub --database:Unit-00 --sqleditor
|
|
|
132
163
|
Execute a specific saved query by name:
|
|
133
164
|
|
|
134
165
|
```bash
|
|
135
|
-
sqlite-hub --database:Unit-00 --sqleditor:"15min Posting Buckets
|
|
166
|
+
sqlite-hub --database:Unit-00 --sqleditor:"15min Posting Buckets without id 96"
|
|
136
167
|
```
|
|
137
168
|
|
|
138
169
|
This searches the query history for the given database, finds the matching saved query by title, executes it, and returns all results with metadata (row count, columns, timing, and data).
|
|
@@ -152,7 +183,7 @@ This searches the query history for the given database, finds the matching saved
|
|
|
152
183
|
| `--database:name --sqleditor` | List all saved queries for a database |
|
|
153
184
|
| `--database:name --sqleditor:"query"` | Execute a saved query by name |
|
|
154
185
|
|
|
155
|
-
###
|
|
186
|
+
### SQL editor CLI example
|
|
156
187
|
|
|
157
188
|

|
|
158
189
|
|
|
@@ -162,7 +193,7 @@ In the screenshot above, you can see a saved query from the SQL editor. You can
|
|
|
162
193
|
sqlite-hub --database:Unit-00 --sqleditor:"Group by creation Year"
|
|
163
194
|
```
|
|
164
195
|
|
|
165
|
-
|
|
196
|
+
Example output:
|
|
166
197
|
|
|
167
198
|
```bash
|
|
168
199
|
Executing: Group by creation Year
|
|
@@ -184,4 +215,4 @@ Results:
|
|
|
184
215
|
|
|
185
216
|
## Changelog
|
|
186
217
|
|
|
187
|
-
[Changelog](/changelog.md)
|
|
218
|
+
[Changelog](./docs/changelog.md)
|
package/database.sqlite
ADDED
|
File without changes
|
package/fill.js
ADDED
|
@@ -0,0 +1,526 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
|
|
3
|
+
const DB_PATH = '/Users/oli/database/project_trump/trump.sqlite';
|
|
4
|
+
const POLYGON_API_KEY = 'j6q5v6iWfVGettyfUxMllrfqg9E9iweT';
|
|
5
|
+
const LIMIT = 1;
|
|
6
|
+
const CALLS_PER_WINDOW = 5;
|
|
7
|
+
const RATE_LIMIT_SLEEP_MS = 61_000;
|
|
8
|
+
|
|
9
|
+
if (!POLYGON_API_KEY) {
|
|
10
|
+
throw new Error(
|
|
11
|
+
"POLYGON_API_KEY fehlt. Starte z.B. mit: POLYGON_API_KEY='dein_key' DB_PATH='./deine-db.sqlite' node fill-mention-impact-prices-all.mjs",
|
|
12
|
+
);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const db = new Database(DB_PATH);
|
|
16
|
+
|
|
17
|
+
let polygonCallCounter = 0;
|
|
18
|
+
const polygonCache = new Map();
|
|
19
|
+
|
|
20
|
+
function sleep(ms) {
|
|
21
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function normalizeUnixSeconds(value) {
|
|
25
|
+
if (value == null) return null;
|
|
26
|
+
|
|
27
|
+
const raw = String(value).trim();
|
|
28
|
+
|
|
29
|
+
// Format: YYYYMMDD, z.B. 20250121
|
|
30
|
+
// Da wir keine genaue Uhrzeit haben, nehmen wir US-Markteröffnung 09:30 New York.
|
|
31
|
+
// Für Januar ist das 14:30 UTC. Für einen ersten Lauf reicht das.
|
|
32
|
+
if (/^\d{8}$/.test(raw)) {
|
|
33
|
+
const year = Number(raw.slice(0, 4));
|
|
34
|
+
const month = Number(raw.slice(4, 6));
|
|
35
|
+
const day = Number(raw.slice(6, 8));
|
|
36
|
+
|
|
37
|
+
return Math.floor(Date.UTC(year, month - 1, day, 14, 30, 0) / 1000);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Format: YYYY-MM-DD
|
|
41
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(raw)) {
|
|
42
|
+
const [year, month, day] = raw.split('-').map(Number);
|
|
43
|
+
|
|
44
|
+
return Math.floor(Date.UTC(year, month - 1, day, 14, 30, 0) / 1000);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const n = Number(value);
|
|
48
|
+
if (!Number.isFinite(n)) return null;
|
|
49
|
+
|
|
50
|
+
// Millisekunden-Timestamp
|
|
51
|
+
if (n > 1_000_000_000_000) {
|
|
52
|
+
return Math.floor(n / 1000);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Unix-Sekunden
|
|
56
|
+
return Math.floor(n);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function dateStringFromUnixSeconds(unixSeconds, offsetDays = 0) {
|
|
60
|
+
const ms = unixSeconds * 1000 + offsetDays * 24 * 60 * 60 * 1000;
|
|
61
|
+
return new Date(ms).toISOString().slice(0, 10);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function etDateKey(ms) {
|
|
65
|
+
const parts = new Intl.DateTimeFormat('en-CA', {
|
|
66
|
+
timeZone: 'America/New_York',
|
|
67
|
+
year: 'numeric',
|
|
68
|
+
month: '2-digit',
|
|
69
|
+
day: '2-digit',
|
|
70
|
+
}).formatToParts(new Date(ms));
|
|
71
|
+
|
|
72
|
+
const map = Object.fromEntries(parts.map(p => [p.type, p.value]));
|
|
73
|
+
return `${map.year}-${map.month}-${map.day}`;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function normalizePolygonSymbol(symbol) {
|
|
77
|
+
if (!symbol) return null;
|
|
78
|
+
|
|
79
|
+
const clean = String(symbol).trim();
|
|
80
|
+
|
|
81
|
+
const map = {
|
|
82
|
+
'^IXIC': 'I:COMP',
|
|
83
|
+
'^GSPC': 'I:SPX',
|
|
84
|
+
'^DJI': 'I:DJI',
|
|
85
|
+
'^NDX': 'I:NDX',
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
return map[clean] || clean;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function waitIfRateLimitWindowHit() {
|
|
92
|
+
if (polygonCallCounter > 0 && polygonCallCounter % CALLS_PER_WINDOW === 0) {
|
|
93
|
+
console.log(`Rate-Limit-Pause nach ${polygonCallCounter} Polygon-Calls: ${RATE_LIMIT_SLEEP_MS / 1000}s`);
|
|
94
|
+
await sleep(RATE_LIMIT_SLEEP_MS);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function fetchPolygonJson(url, label) {
|
|
99
|
+
polygonCallCounter += 1;
|
|
100
|
+
|
|
101
|
+
console.log(`Polygon call #${polygonCallCounter}: ${label}`);
|
|
102
|
+
|
|
103
|
+
const res = await fetch(url);
|
|
104
|
+
const text = await res.text();
|
|
105
|
+
|
|
106
|
+
await waitIfRateLimitWindowHit();
|
|
107
|
+
|
|
108
|
+
if (res.status === 429) {
|
|
109
|
+
console.warn('Polygon 429 Rate Limit. Warte 61 Sekunden und versuche es nochmal...');
|
|
110
|
+
await sleep(RATE_LIMIT_SLEEP_MS);
|
|
111
|
+
return fetchPolygonJson(url, `${label} retry`);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (!res.ok) {
|
|
115
|
+
throw new Error(`Polygon HTTP ${res.status}: ${text.slice(0, 500)}`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
let json;
|
|
119
|
+
|
|
120
|
+
try {
|
|
121
|
+
json = JSON.parse(text);
|
|
122
|
+
} catch {
|
|
123
|
+
throw new Error(`Polygon response ist kein JSON: ${text.slice(0, 500)}`);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (json.status === 'ERROR') {
|
|
127
|
+
throw new Error(`Polygon ERROR: ${json.error || JSON.stringify(json).slice(0, 500)}`);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return json;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function polygonAggs(symbol, multiplier, timespan, from, to) {
|
|
134
|
+
const polygonSymbol = normalizePolygonSymbol(symbol);
|
|
135
|
+
|
|
136
|
+
if (!polygonSymbol) {
|
|
137
|
+
throw new Error('Kein Symbol vorhanden.');
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const cacheKey = [polygonSymbol, multiplier, timespan, from, to].join('|');
|
|
141
|
+
|
|
142
|
+
if (polygonCache.has(cacheKey)) {
|
|
143
|
+
console.log(`Cache hit: ${cacheKey}`);
|
|
144
|
+
return polygonCache.get(cacheKey);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const url = new URL(
|
|
148
|
+
`https://api.polygon.io/v2/aggs/ticker/${encodeURIComponent(polygonSymbol)}/range/${multiplier}/${timespan}/${from}/${to}`,
|
|
149
|
+
);
|
|
150
|
+
|
|
151
|
+
url.searchParams.set('adjusted', 'true');
|
|
152
|
+
url.searchParams.set('sort', 'asc');
|
|
153
|
+
url.searchParams.set('limit', '50000');
|
|
154
|
+
url.searchParams.set('apiKey', POLYGON_API_KEY);
|
|
155
|
+
|
|
156
|
+
const json = await fetchPolygonJson(url, `${polygonSymbol} ${multiplier}/${timespan} ${from} -> ${to}`);
|
|
157
|
+
|
|
158
|
+
const bars = (json.results || [])
|
|
159
|
+
.filter(row => row && Number.isFinite(row.t) && Number.isFinite(row.c))
|
|
160
|
+
.map(row => ({
|
|
161
|
+
t: row.t,
|
|
162
|
+
open: row.o,
|
|
163
|
+
high: row.h,
|
|
164
|
+
low: row.l,
|
|
165
|
+
close: row.c,
|
|
166
|
+
volume: row.v,
|
|
167
|
+
}))
|
|
168
|
+
.sort((a, b) => a.t - b.t);
|
|
169
|
+
|
|
170
|
+
polygonCache.set(cacheKey, bars);
|
|
171
|
+
|
|
172
|
+
return bars;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function firstBarAtOrAfter(bars, targetMs) {
|
|
176
|
+
return bars.find(bar => bar.t >= targetMs && Number.isFinite(bar.close)) || null;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function eodCloseForSession(intradayBars, sessionDateKey) {
|
|
180
|
+
const sessionBars = intradayBars.filter(bar => etDateKey(bar.t) === sessionDateKey);
|
|
181
|
+
|
|
182
|
+
if (sessionBars.length === 0) {
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
return sessionBars[sessionBars.length - 1].close;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function dailyRows(dailyBars) {
|
|
190
|
+
return dailyBars
|
|
191
|
+
.filter(bar => Number.isFinite(bar.close))
|
|
192
|
+
.map(bar => ({
|
|
193
|
+
t: bar.t,
|
|
194
|
+
dateKey: etDateKey(bar.t),
|
|
195
|
+
close: bar.close,
|
|
196
|
+
}))
|
|
197
|
+
.sort((a, b) => a.t - b.t);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function pickPrices(intradayBars, dailyBars, eventUnixSeconds) {
|
|
201
|
+
const eventMs = eventUnixSeconds * 1000;
|
|
202
|
+
|
|
203
|
+
const atEventBar = firstBarAtOrAfter(intradayBars, eventMs);
|
|
204
|
+
|
|
205
|
+
if (!atEventBar) {
|
|
206
|
+
return {
|
|
207
|
+
price_at_event: null,
|
|
208
|
+
price_1h: null,
|
|
209
|
+
price_eod: null,
|
|
210
|
+
price_next_trading_day_1: null,
|
|
211
|
+
price_next_trading_day_2: null,
|
|
212
|
+
price_next_trading_day_3: null,
|
|
213
|
+
price_next_trading_day_5: null,
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const oneHourBar = firstBarAtOrAfter(intradayBars, atEventBar.t + 60 * 60 * 1000);
|
|
218
|
+
|
|
219
|
+
const baseSessionDateKey = etDateKey(atEventBar.t);
|
|
220
|
+
const eod = eodCloseForSession(intradayBars, baseSessionDateKey);
|
|
221
|
+
|
|
222
|
+
const days = dailyRows(dailyBars);
|
|
223
|
+
|
|
224
|
+
let baseDayIndex = days.findIndex(day => day.dateKey === baseSessionDateKey);
|
|
225
|
+
|
|
226
|
+
if (baseDayIndex === -1) {
|
|
227
|
+
baseDayIndex = days.findIndex(day => day.t >= atEventBar.t);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function nextTradingDayClose(n) {
|
|
231
|
+
if (baseDayIndex === -1) return null;
|
|
232
|
+
const row = days[baseDayIndex + n];
|
|
233
|
+
return row ? row.close : null;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
return {
|
|
237
|
+
price_at_event: atEventBar.close,
|
|
238
|
+
price_1h: oneHourBar ? oneHourBar.close : null,
|
|
239
|
+
price_eod: eod,
|
|
240
|
+
price_next_trading_day_1: nextTradingDayClose(1),
|
|
241
|
+
price_next_trading_day_2: nextTradingDayClose(2),
|
|
242
|
+
price_next_trading_day_3: nextTradingDayClose(3),
|
|
243
|
+
price_next_trading_day_5: nextTradingDayClose(5),
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function calcReturn(fromPrice, toPrice) {
|
|
248
|
+
if (!Number.isFinite(fromPrice)) return null;
|
|
249
|
+
if (!Number.isFinite(toPrice)) return null;
|
|
250
|
+
if (fromPrice === 0) return null;
|
|
251
|
+
|
|
252
|
+
return (toPrice - fromPrice) / fromPrice;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function calcReturns(prices) {
|
|
256
|
+
return {
|
|
257
|
+
return_1h: calcReturn(prices.price_at_event, prices.price_1h),
|
|
258
|
+
return_eod: calcReturn(prices.price_at_event, prices.price_eod),
|
|
259
|
+
return_next_trading_day_1: calcReturn(prices.price_at_event, prices.price_next_trading_day_1),
|
|
260
|
+
return_next_trading_day_2: calcReturn(prices.price_at_event, prices.price_next_trading_day_2),
|
|
261
|
+
return_next_trading_day_3: calcReturn(prices.price_at_event, prices.price_next_trading_day_3),
|
|
262
|
+
return_next_trading_day_5: calcReturn(prices.price_at_event, prices.price_next_trading_day_5),
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function calcDelta(stockReturn, benchmarkReturn) {
|
|
267
|
+
if (!Number.isFinite(stockReturn)) return null;
|
|
268
|
+
if (!Number.isFinite(benchmarkReturn)) return null;
|
|
269
|
+
|
|
270
|
+
return stockReturn - benchmarkReturn;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function completionStatus(stockPrices, benchmarkPrices) {
|
|
274
|
+
const required = [
|
|
275
|
+
stockPrices.price_at_event,
|
|
276
|
+
stockPrices.price_1h,
|
|
277
|
+
stockPrices.price_eod,
|
|
278
|
+
stockPrices.price_next_trading_day_1,
|
|
279
|
+
stockPrices.price_next_trading_day_2,
|
|
280
|
+
stockPrices.price_next_trading_day_3,
|
|
281
|
+
stockPrices.price_next_trading_day_5,
|
|
282
|
+
|
|
283
|
+
benchmarkPrices.price_at_event,
|
|
284
|
+
benchmarkPrices.price_1h,
|
|
285
|
+
benchmarkPrices.price_eod,
|
|
286
|
+
benchmarkPrices.price_next_trading_day_1,
|
|
287
|
+
benchmarkPrices.price_next_trading_day_2,
|
|
288
|
+
benchmarkPrices.price_next_trading_day_3,
|
|
289
|
+
benchmarkPrices.price_next_trading_day_5,
|
|
290
|
+
];
|
|
291
|
+
|
|
292
|
+
return required.every(Number.isFinite) ? 'complete' : 'partial';
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
const selectRows = db.prepare(`
|
|
296
|
+
SELECT
|
|
297
|
+
mi.id,
|
|
298
|
+
mi.event_time,
|
|
299
|
+
COALESCE(mi.benchmark_symbol, '^IXIC') AS benchmark_symbol,
|
|
300
|
+
c.ticker AS stock_symbol,
|
|
301
|
+
c.short_name AS company_name
|
|
302
|
+
FROM mention_impact mi
|
|
303
|
+
JOIN companies c
|
|
304
|
+
ON c.id = mi.company_id
|
|
305
|
+
WHERE mi.event_time IS NOT NULL
|
|
306
|
+
AND c.ticker IS NOT NULL
|
|
307
|
+
AND TRIM(c.ticker) != ''
|
|
308
|
+
AND (
|
|
309
|
+
mi.price_at_event IS NULL
|
|
310
|
+
OR mi.benchmark_price_at_event IS NULL
|
|
311
|
+
OR mi.status IN ('pending', 'failed')
|
|
312
|
+
)
|
|
313
|
+
ORDER BY mi.event_time ASC
|
|
314
|
+
`);
|
|
315
|
+
|
|
316
|
+
const updateRow = db.prepare(`
|
|
317
|
+
UPDATE mention_impact
|
|
318
|
+
SET
|
|
319
|
+
price_at_event = @price_at_event,
|
|
320
|
+
price_1h = @price_1h,
|
|
321
|
+
price_eod = @price_eod,
|
|
322
|
+
price_next_trading_day_1 = @price_next_trading_day_1,
|
|
323
|
+
price_next_trading_day_2 = @price_next_trading_day_2,
|
|
324
|
+
price_next_trading_day_3 = @price_next_trading_day_3,
|
|
325
|
+
price_next_trading_day_5 = @price_next_trading_day_5,
|
|
326
|
+
|
|
327
|
+
benchmark_price_at_event = @benchmark_price_at_event,
|
|
328
|
+
benchmark_price_1h = @benchmark_price_1h,
|
|
329
|
+
benchmark_price_eod = @benchmark_price_eod,
|
|
330
|
+
benchmark_price_next_trading_day_1 = @benchmark_price_next_trading_day_1,
|
|
331
|
+
benchmark_price_next_trading_day_2 = @benchmark_price_next_trading_day_2,
|
|
332
|
+
benchmark_price_next_trading_day_3 = @benchmark_price_next_trading_day_3,
|
|
333
|
+
benchmark_price_next_trading_day_5 = @benchmark_price_next_trading_day_5,
|
|
334
|
+
|
|
335
|
+
stock_return_1h = @stock_return_1h,
|
|
336
|
+
stock_return_eod = @stock_return_eod,
|
|
337
|
+
stock_return_next_trading_day_1 = @stock_return_next_trading_day_1,
|
|
338
|
+
stock_return_next_trading_day_2 = @stock_return_next_trading_day_2,
|
|
339
|
+
stock_return_next_trading_day_3 = @stock_return_next_trading_day_3,
|
|
340
|
+
stock_return_next_trading_day_5 = @stock_return_next_trading_day_5,
|
|
341
|
+
|
|
342
|
+
benchmark_return_1h = @benchmark_return_1h,
|
|
343
|
+
benchmark_return_eod = @benchmark_return_eod,
|
|
344
|
+
benchmark_return_next_trading_day_1 = @benchmark_return_next_trading_day_1,
|
|
345
|
+
benchmark_return_next_trading_day_2 = @benchmark_return_next_trading_day_2,
|
|
346
|
+
benchmark_return_next_trading_day_3 = @benchmark_return_next_trading_day_3,
|
|
347
|
+
benchmark_return_next_trading_day_5 = @benchmark_return_next_trading_day_5,
|
|
348
|
+
|
|
349
|
+
delta_return_1h = @delta_return_1h,
|
|
350
|
+
delta_return_eod = @delta_return_eod,
|
|
351
|
+
delta_return_next_trading_day_1 = @delta_return_next_trading_day_1,
|
|
352
|
+
delta_return_next_trading_day_2 = @delta_return_next_trading_day_2,
|
|
353
|
+
delta_return_next_trading_day_3 = @delta_return_next_trading_day_3,
|
|
354
|
+
delta_return_next_trading_day_5 = @delta_return_next_trading_day_5,
|
|
355
|
+
|
|
356
|
+
status = @status,
|
|
357
|
+
error_text = NULL,
|
|
358
|
+
updated_at = unixepoch()
|
|
359
|
+
WHERE id = @id
|
|
360
|
+
`);
|
|
361
|
+
|
|
362
|
+
const markFailed = db.prepare(`
|
|
363
|
+
UPDATE mention_impact
|
|
364
|
+
SET
|
|
365
|
+
status = 'failed',
|
|
366
|
+
error_text = @error_text,
|
|
367
|
+
updated_at = unixepoch()
|
|
368
|
+
WHERE id = @id
|
|
369
|
+
`);
|
|
370
|
+
|
|
371
|
+
async function fillOne(row) {
|
|
372
|
+
const eventUnixSeconds = normalizeUnixSeconds(row.event_time);
|
|
373
|
+
|
|
374
|
+
if (!eventUnixSeconds) {
|
|
375
|
+
throw new Error('event_time ist leer oder ungültig.');
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
const stockSymbol = normalizePolygonSymbol(row.stock_symbol);
|
|
379
|
+
const benchmarkSymbol = normalizePolygonSymbol(row.benchmark_symbol || '^IXIC');
|
|
380
|
+
|
|
381
|
+
if (/^\d+$/.test(String(stockSymbol))) {
|
|
382
|
+
throw new Error(
|
|
383
|
+
`Ticker "${stockSymbol}" sieht nach einem nicht-US-Ticker aus. Polygon braucht ein handelbares Symbol wie AAPL, MSFT, TSLA oder ggf. ein ADR/OTC-Symbol.`,
|
|
384
|
+
);
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
if (!stockSymbol) {
|
|
388
|
+
throw new Error('Kein Aktien-Ticker in companies.ticker vorhanden.');
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
if (!benchmarkSymbol) {
|
|
392
|
+
throw new Error('Kein Benchmark-Symbol vorhanden.');
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const from = dateStringFromUnixSeconds(eventUnixSeconds, -3);
|
|
396
|
+
const to = dateStringFromUnixSeconds(eventUnixSeconds, 14);
|
|
397
|
+
|
|
398
|
+
const stockIntradayBars = await polygonAggs(stockSymbol, 5, 'minute', from, to);
|
|
399
|
+
const stockDailyBars = await polygonAggs(stockSymbol, 1, 'day', from, to);
|
|
400
|
+
|
|
401
|
+
const benchmarkIntradayBars = await polygonAggs(benchmarkSymbol, 5, 'minute', from, to);
|
|
402
|
+
const benchmarkDailyBars = await polygonAggs(benchmarkSymbol, 1, 'day', from, to);
|
|
403
|
+
|
|
404
|
+
const stockPrices = pickPrices(stockIntradayBars, stockDailyBars, eventUnixSeconds);
|
|
405
|
+
const benchmarkPrices = pickPrices(benchmarkIntradayBars, benchmarkDailyBars, eventUnixSeconds);
|
|
406
|
+
|
|
407
|
+
const stockReturns = calcReturns(stockPrices);
|
|
408
|
+
const benchmarkReturns = calcReturns(benchmarkPrices);
|
|
409
|
+
|
|
410
|
+
const payload = {
|
|
411
|
+
id: row.id,
|
|
412
|
+
|
|
413
|
+
price_at_event: stockPrices.price_at_event,
|
|
414
|
+
price_1h: stockPrices.price_1h,
|
|
415
|
+
price_eod: stockPrices.price_eod,
|
|
416
|
+
price_next_trading_day_1: stockPrices.price_next_trading_day_1,
|
|
417
|
+
price_next_trading_day_2: stockPrices.price_next_trading_day_2,
|
|
418
|
+
price_next_trading_day_3: stockPrices.price_next_trading_day_3,
|
|
419
|
+
price_next_trading_day_5: stockPrices.price_next_trading_day_5,
|
|
420
|
+
|
|
421
|
+
benchmark_price_at_event: benchmarkPrices.price_at_event,
|
|
422
|
+
benchmark_price_1h: benchmarkPrices.price_1h,
|
|
423
|
+
benchmark_price_eod: benchmarkPrices.price_eod,
|
|
424
|
+
benchmark_price_next_trading_day_1: benchmarkPrices.price_next_trading_day_1,
|
|
425
|
+
benchmark_price_next_trading_day_2: benchmarkPrices.price_next_trading_day_2,
|
|
426
|
+
benchmark_price_next_trading_day_3: benchmarkPrices.price_next_trading_day_3,
|
|
427
|
+
benchmark_price_next_trading_day_5: benchmarkPrices.price_next_trading_day_5,
|
|
428
|
+
|
|
429
|
+
stock_return_1h: stockReturns.return_1h,
|
|
430
|
+
stock_return_eod: stockReturns.return_eod,
|
|
431
|
+
stock_return_next_trading_day_1: stockReturns.return_next_trading_day_1,
|
|
432
|
+
stock_return_next_trading_day_2: stockReturns.return_next_trading_day_2,
|
|
433
|
+
stock_return_next_trading_day_3: stockReturns.return_next_trading_day_3,
|
|
434
|
+
stock_return_next_trading_day_5: stockReturns.return_next_trading_day_5,
|
|
435
|
+
|
|
436
|
+
benchmark_return_1h: benchmarkReturns.return_1h,
|
|
437
|
+
benchmark_return_eod: benchmarkReturns.return_eod,
|
|
438
|
+
benchmark_return_next_trading_day_1: benchmarkReturns.return_next_trading_day_1,
|
|
439
|
+
benchmark_return_next_trading_day_2: benchmarkReturns.return_next_trading_day_2,
|
|
440
|
+
benchmark_return_next_trading_day_3: benchmarkReturns.return_next_trading_day_3,
|
|
441
|
+
benchmark_return_next_trading_day_5: benchmarkReturns.return_next_trading_day_5,
|
|
442
|
+
|
|
443
|
+
delta_return_1h: calcDelta(stockReturns.return_1h, benchmarkReturns.return_1h),
|
|
444
|
+
delta_return_eod: calcDelta(stockReturns.return_eod, benchmarkReturns.return_eod),
|
|
445
|
+
|
|
446
|
+
delta_return_next_trading_day_1: calcDelta(
|
|
447
|
+
stockReturns.return_next_trading_day_1,
|
|
448
|
+
benchmarkReturns.return_next_trading_day_1,
|
|
449
|
+
),
|
|
450
|
+
|
|
451
|
+
delta_return_next_trading_day_2: calcDelta(
|
|
452
|
+
stockReturns.return_next_trading_day_2,
|
|
453
|
+
benchmarkReturns.return_next_trading_day_2,
|
|
454
|
+
),
|
|
455
|
+
|
|
456
|
+
delta_return_next_trading_day_3: calcDelta(
|
|
457
|
+
stockReturns.return_next_trading_day_3,
|
|
458
|
+
benchmarkReturns.return_next_trading_day_3,
|
|
459
|
+
),
|
|
460
|
+
|
|
461
|
+
delta_return_next_trading_day_5: calcDelta(
|
|
462
|
+
stockReturns.return_next_trading_day_5,
|
|
463
|
+
benchmarkReturns.return_next_trading_day_5,
|
|
464
|
+
),
|
|
465
|
+
|
|
466
|
+
status: completionStatus(stockPrices, benchmarkPrices),
|
|
467
|
+
};
|
|
468
|
+
|
|
469
|
+
updateRow.run(payload);
|
|
470
|
+
|
|
471
|
+
return payload;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
async function main() {
|
|
475
|
+
const rows = selectRows.all();
|
|
476
|
+
|
|
477
|
+
console.log(`DB: ${DB_PATH}`);
|
|
478
|
+
console.log(`Gefundene mention_impact rows: ${rows.length}`);
|
|
479
|
+
console.log(`Rate limit: ${CALLS_PER_WINDOW} Calls, dann ${RATE_LIMIT_SLEEP_MS / 1000}s Pause`);
|
|
480
|
+
|
|
481
|
+
let processed = 0;
|
|
482
|
+
let failed = 0;
|
|
483
|
+
|
|
484
|
+
for (const row of rows) {
|
|
485
|
+
try {
|
|
486
|
+
processed += 1;
|
|
487
|
+
|
|
488
|
+
console.log('');
|
|
489
|
+
console.log(`Bearbeite ${processed}/${rows.length}: ${row.id}`);
|
|
490
|
+
console.log(`Company: ${row.company_name}`);
|
|
491
|
+
console.log(`Ticker: ${row.stock_symbol}`);
|
|
492
|
+
console.log(`Benchmark: ${row.benchmark_symbol}`);
|
|
493
|
+
console.log(`Event time: ${row.event_time}`);
|
|
494
|
+
|
|
495
|
+
const payload = await fillOne(row);
|
|
496
|
+
|
|
497
|
+
console.log(`OK: ${row.id} -> ${payload.status}`);
|
|
498
|
+
console.log(`price_at_event: ${payload.price_at_event}`);
|
|
499
|
+
console.log(`price_1h: ${payload.price_1h}`);
|
|
500
|
+
console.log(`price_eod: ${payload.price_eod}`);
|
|
501
|
+
console.log(`stock_return_1h: ${payload.stock_return_1h}`);
|
|
502
|
+
console.log(`delta_return_1h: ${payload.delta_return_1h}`);
|
|
503
|
+
} catch (error) {
|
|
504
|
+
failed += 1;
|
|
505
|
+
|
|
506
|
+
console.error(`FAILED: ${row.id}`);
|
|
507
|
+
console.error(error.message);
|
|
508
|
+
|
|
509
|
+
markFailed.run({
|
|
510
|
+
id: row.id,
|
|
511
|
+
error_text: error.message,
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
console.log('');
|
|
517
|
+
console.log('Fertig.');
|
|
518
|
+
console.log(`Verarbeitet: ${processed}`);
|
|
519
|
+
console.log(`Fehlgeschlagen: ${failed}`);
|
|
520
|
+
console.log(`Polygon-Calls insgesamt: ${polygonCallCounter}`);
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
main().catch(error => {
|
|
524
|
+
console.error(error);
|
|
525
|
+
process.exit(1);
|
|
526
|
+
});
|