skewtape 0.0.1 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +182 -0
- package/dist/cjs/backtest.d.ts +146 -0
- package/dist/cjs/backtest.js +410 -0
- package/dist/cjs/client.d.ts +255 -0
- package/dist/cjs/client.js +447 -0
- package/dist/cjs/download.d.ts +33 -0
- package/dist/cjs/download.js +160 -0
- package/dist/cjs/errors.d.ts +76 -0
- package/dist/cjs/errors.js +159 -0
- package/dist/cjs/exact.d.ts +36 -0
- package/dist/cjs/exact.js +168 -0
- package/dist/cjs/fees.d.ts +68 -0
- package/dist/cjs/fees.js +196 -0
- package/dist/cjs/geoblock.d.ts +48 -0
- package/dist/cjs/geoblock.js +78 -0
- package/dist/cjs/index.d.ts +31 -0
- package/dist/cjs/index.js +98 -0
- package/dist/cjs/kalshi.d.ts +261 -0
- package/dist/cjs/kalshi.js +416 -0
- package/dist/cjs/keccak.d.ts +10 -0
- package/dist/cjs/keccak.js +84 -0
- package/dist/cjs/msgpack.d.ts +15 -0
- package/dist/cjs/msgpack.js +112 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/paginated.d.ts +26 -0
- package/dist/cjs/paginated.js +51 -0
- package/dist/cjs/quote-math.d.ts +146 -0
- package/dist/cjs/quote-math.js +446 -0
- package/dist/cjs/quote.d.ts +108 -0
- package/dist/cjs/quote.js +307 -0
- package/dist/cjs/route-hyperliquid.d.ts +228 -0
- package/dist/cjs/route-hyperliquid.js +387 -0
- package/dist/cjs/route.d.ts +297 -0
- package/dist/cjs/route.js +516 -0
- package/dist/cjs/stream.d.ts +99 -0
- package/dist/cjs/stream.js +529 -0
- package/dist/cjs/version.d.ts +6 -0
- package/dist/cjs/version.js +9 -0
- package/dist/esm/backtest.d.ts +146 -0
- package/dist/esm/backtest.js +403 -0
- package/dist/esm/client.d.ts +255 -0
- package/dist/esm/client.js +441 -0
- package/dist/esm/download.d.ts +33 -0
- package/dist/esm/download.js +155 -0
- package/dist/esm/errors.d.ts +76 -0
- package/dist/esm/errors.js +142 -0
- package/dist/esm/exact.d.ts +36 -0
- package/dist/esm/exact.js +158 -0
- package/dist/esm/fees.d.ts +68 -0
- package/dist/esm/fees.js +185 -0
- package/dist/esm/geoblock.d.ts +48 -0
- package/dist/esm/geoblock.js +74 -0
- package/dist/esm/index.d.ts +31 -0
- package/dist/esm/index.js +30 -0
- package/dist/esm/kalshi.d.ts +261 -0
- package/dist/esm/kalshi.js +404 -0
- package/dist/esm/keccak.d.ts +10 -0
- package/dist/esm/keccak.js +80 -0
- package/dist/esm/msgpack.d.ts +15 -0
- package/dist/esm/msgpack.js +109 -0
- package/dist/esm/paginated.d.ts +26 -0
- package/dist/esm/paginated.js +47 -0
- package/dist/esm/quote-math.d.ts +146 -0
- package/dist/esm/quote-math.js +437 -0
- package/dist/esm/quote.d.ts +108 -0
- package/dist/esm/quote.js +297 -0
- package/dist/esm/route-hyperliquid.d.ts +228 -0
- package/dist/esm/route-hyperliquid.js +376 -0
- package/dist/esm/route.d.ts +297 -0
- package/dist/esm/route.js +507 -0
- package/dist/esm/stream.d.ts +99 -0
- package/dist/esm/stream.js +524 -0
- package/dist/esm/version.d.ts +6 -0
- package/dist/esm/version.js +6 -0
- package/package.json +48 -1
- package/src/backtest.ts +490 -0
- package/src/client.ts +595 -0
- package/src/download.ts +161 -0
- package/src/errors.ts +143 -0
- package/src/exact.ts +159 -0
- package/src/fees.ts +189 -0
- package/src/generated/schema.d.ts +2500 -0
- package/src/geoblock.ts +90 -0
- package/src/index.ts +146 -0
- package/src/kalshi.ts +563 -0
- package/src/keccak.ts +82 -0
- package/src/msgpack.ts +83 -0
- package/src/paginated.ts +51 -0
- package/src/quote-math.ts +555 -0
- package/src/quote.ts +324 -0
- package/src/route-hyperliquid.ts +457 -0
- package/src/route.ts +631 -0
- package/src/stream.ts +539 -0
- package/src/version.ts +6 -0
|
@@ -0,0 +1,441 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The facade: sk.markets, sk.events (and sk.events.get), sk.pairs, sk.fills,
|
|
3
|
+
* sk.candles, sk.resolutions, sk.fees, sk.venues, sk.download and sk.quote
|
|
4
|
+
* (quote.ts), over the openapi-fetch client typed by the generated schema.
|
|
5
|
+
* Lists come back as a Paginated: iterate (for await) for every item across
|
|
6
|
+
* pages, .pages() for the pages themselves, .first() for the first item. A 429
|
|
7
|
+
* rate_limited is retried with the server's Retry-After; every other error, a
|
|
8
|
+
* quota_exceeded 429 included, is a SkewtapeError with the envelope's fields,
|
|
9
|
+
* thrown at once.
|
|
10
|
+
* sk.kalshi() is the optional live Kalshi client (./kalshi.ts), apart from
|
|
11
|
+
* the API. sk.route builds an unsigned order with the developer's own code
|
|
12
|
+
* (./route.ts) and sk.geoblock asks a venue's own geoblock check
|
|
13
|
+
* (./geoblock.ts); neither sends the API key.
|
|
14
|
+
*/
|
|
15
|
+
import { resolve, sep } from "node:path";
|
|
16
|
+
import createOpenapiClient from "openapi-fetch";
|
|
17
|
+
import { CANDLE_MICROS, runBacktest, settledAt } from "./backtest.js";
|
|
18
|
+
import { downloadFile, holds, verifyChecksums } from "./download.js";
|
|
19
|
+
import { ConfigurationError, NotFoundError, SkewtapeError, errorFromResponse, retryAfterSeconds } from "./errors.js";
|
|
20
|
+
import { Stream, feedUrl } from "./stream.js";
|
|
21
|
+
import { geoblock } from "./geoblock.js";
|
|
22
|
+
import { route } from "./route.js";
|
|
23
|
+
import { parseMicros } from "./exact.js";
|
|
24
|
+
import { Kalshi } from "./kalshi.js";
|
|
25
|
+
import { Paginated } from "./paginated.js";
|
|
26
|
+
import { quoteFromContext } from "./quote.js";
|
|
27
|
+
import { VERSION } from "./version.js";
|
|
28
|
+
export { Paginated } from "./paginated.js";
|
|
29
|
+
export const DEFAULT_BASE_URL = "https://api.skewtape.com";
|
|
30
|
+
export const ENV_API_KEY = "SKEWTAPE_API_KEY";
|
|
31
|
+
/**
|
|
32
|
+
* The windows [t - lookback, t + 1 ms) up to each time (microseconds), as the
|
|
33
|
+
* API's exclusive to reads them: dateOf drops the microseconds, so the next
|
|
34
|
+
* millisecond keeps a row at t itself in. Overlapping ones merged.
|
|
35
|
+
*/
|
|
36
|
+
function windows(times, lookback) {
|
|
37
|
+
const spans = times.map((t) => [t - lookback, t + 1000n]).sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
|
|
38
|
+
const merged = [];
|
|
39
|
+
for (const [start, end] of spans) {
|
|
40
|
+
const last = merged.at(-1);
|
|
41
|
+
if (last && start <= last[1])
|
|
42
|
+
last[1] = end > last[1] ? end : last[1];
|
|
43
|
+
else
|
|
44
|
+
merged.push([start, end]);
|
|
45
|
+
}
|
|
46
|
+
return merged;
|
|
47
|
+
}
|
|
48
|
+
function dateOf(micros) {
|
|
49
|
+
return new Date(Number(micros / 1000n));
|
|
50
|
+
}
|
|
51
|
+
function iso(v) {
|
|
52
|
+
return v instanceof Date ? v.toISOString() : v;
|
|
53
|
+
}
|
|
54
|
+
function defaultSleep(seconds) {
|
|
55
|
+
return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
|
|
56
|
+
}
|
|
57
|
+
/** How many of an event's members are active. */
|
|
58
|
+
export function activeMembers(event) {
|
|
59
|
+
return event.members.filter((m) => m.status === "active").length;
|
|
60
|
+
}
|
|
61
|
+
/** The envelope's code in an openapi-fetch error body, when it is one. */
|
|
62
|
+
function envelopeCode(error) {
|
|
63
|
+
if (error === null || typeof error !== "object" || !("error" in error))
|
|
64
|
+
return undefined;
|
|
65
|
+
const e = error.error;
|
|
66
|
+
return e !== null && typeof e === "object" && typeof e.code === "string" ? e.code : undefined;
|
|
67
|
+
}
|
|
68
|
+
/** A fetch that gives up after timeout seconds, unless the caller brought its own abort handle. */
|
|
69
|
+
function withTimeout(fetchImpl, timeout) {
|
|
70
|
+
return (input, init) => fetchImpl(input, { ...init, signal: init?.signal ?? AbortSignal.timeout(timeout * 1000) });
|
|
71
|
+
}
|
|
72
|
+
export class Skewtape {
|
|
73
|
+
baseUrl;
|
|
74
|
+
maxRetries;
|
|
75
|
+
timeout;
|
|
76
|
+
api;
|
|
77
|
+
sleep;
|
|
78
|
+
userSleep;
|
|
79
|
+
/** A private field: inspecting or logging the client does not show the key. */
|
|
80
|
+
#apiKey;
|
|
81
|
+
constructor(options = {}) {
|
|
82
|
+
const key = options.apiKey ?? process.env[ENV_API_KEY] ?? "";
|
|
83
|
+
if (!key)
|
|
84
|
+
throw new ConfigurationError(`no API key: pass apiKey or set ${ENV_API_KEY}`);
|
|
85
|
+
if (options.maxRetries !== undefined && options.maxRetries < 0)
|
|
86
|
+
throw new ConfigurationError("maxRetries must not be negative");
|
|
87
|
+
if (options.timeout !== undefined && !(options.timeout > 0))
|
|
88
|
+
throw new ConfigurationError("timeout is seconds, above zero");
|
|
89
|
+
this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
|
|
90
|
+
this.#apiKey = key;
|
|
91
|
+
this.maxRetries = options.maxRetries ?? 3;
|
|
92
|
+
this.timeout = options.timeout ?? 30;
|
|
93
|
+
this.sleep = options.sleep ?? defaultSleep;
|
|
94
|
+
this.userSleep = options.sleep;
|
|
95
|
+
this.api = createOpenapiClient({
|
|
96
|
+
baseUrl: this.baseUrl,
|
|
97
|
+
fetch: withTimeout(options.fetch ?? fetch, this.timeout),
|
|
98
|
+
headers: { "X-API-Key": key, "User-Agent": `skewtape-ts/${VERSION}` },
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Run one request: a 429 whose envelope says rate_limited (or carries no
|
|
103
|
+
* envelope, as a proxy's would) retried up to maxRetries times after the
|
|
104
|
+
* server's Retry-After (one second without it); a 2xx returned as parsed;
|
|
105
|
+
* anything else, a quota_exceeded 429 included, thrown at once from the
|
|
106
|
+
* error envelope. A request past the timeout is a SkewtapeError.
|
|
107
|
+
*/
|
|
108
|
+
async call(run) {
|
|
109
|
+
for (let attempt = 0;; attempt++) {
|
|
110
|
+
let result;
|
|
111
|
+
try {
|
|
112
|
+
result = await run();
|
|
113
|
+
}
|
|
114
|
+
catch (err) {
|
|
115
|
+
if (err instanceof Error && (err.name === "TimeoutError" || err.name === "AbortError")) {
|
|
116
|
+
throw new SkewtapeError(`the request took longer than ${this.timeout} s`);
|
|
117
|
+
}
|
|
118
|
+
throw err;
|
|
119
|
+
}
|
|
120
|
+
const { data, error, response } = result;
|
|
121
|
+
if (response.ok) {
|
|
122
|
+
if (data === undefined)
|
|
123
|
+
throw new SkewtapeError(`HTTP ${response.status} with no body the client knows`, { status: response.status });
|
|
124
|
+
return data;
|
|
125
|
+
}
|
|
126
|
+
const code = envelopeCode(error);
|
|
127
|
+
if (response.status === 429 && (code === "rate_limited" || code === undefined) && attempt < this.maxRetries) {
|
|
128
|
+
await this.sleep(retryAfterSeconds(response.headers) ?? 1);
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
throw errorFromResponse(response.status, response.headers, error);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/** sk.markets({venue, resolved}) lists markets; sk.markets.get(venue, marketId) reads one. */
|
|
135
|
+
markets = Object.assign((options = {}) => new Paginated(async (cursor) => {
|
|
136
|
+
const query = {
|
|
137
|
+
venue: options.venue,
|
|
138
|
+
market_id: options.marketId,
|
|
139
|
+
updated_since: iso(options.updatedSince),
|
|
140
|
+
limit: options.limit ?? 100,
|
|
141
|
+
cursor,
|
|
142
|
+
};
|
|
143
|
+
const data = await this.call(() => this.api.GET("/v1/markets", { params: { query } }));
|
|
144
|
+
return { items: data.data, nextCursor: data.next_cursor ?? null };
|
|
145
|
+
}, options.resolved === undefined ? undefined : (m) => (m.status === "resolved") === options.resolved), {
|
|
146
|
+
get: (venue, marketId) => this.call(() => this.api.GET("/v1/markets/{venue}/{market_id}", { params: { path: { venue, market_id: marketId } } })),
|
|
147
|
+
});
|
|
148
|
+
/** sk.events({minConfidence, resolutionEqual}) lists matched events; sk.events.get(eventId) reads one. */
|
|
149
|
+
events = Object.assign((options = {}) => new Paginated(this.eventsPage(options)), {
|
|
150
|
+
get: (eventId) => this.call(() => this.api.GET("/v1/matched_events/{event_id}", { params: { path: { event_id: eventId } } })),
|
|
151
|
+
});
|
|
152
|
+
/** sk.pairs(...) is sk.events(...) kept to the pairs: a pair is a matched event with exactly two active members. sk.pairs.get refuses, as a 404 would, any other. */
|
|
153
|
+
pairs = Object.assign((options = {}) => new Paginated(this.eventsPage(options), (e) => activeMembers(e) === 2), {
|
|
154
|
+
get: async (eventId) => {
|
|
155
|
+
const event = await this.events.get(eventId);
|
|
156
|
+
const n = activeMembers(event);
|
|
157
|
+
if (n !== 2)
|
|
158
|
+
throw new NotFoundError(`${eventId} is not a pair: ${n} active members`, { status: 404, code: "not_found" });
|
|
159
|
+
return event;
|
|
160
|
+
},
|
|
161
|
+
});
|
|
162
|
+
eventsPage(options) {
|
|
163
|
+
return async (cursor) => {
|
|
164
|
+
const query = {
|
|
165
|
+
min_confidence: options.minConfidence,
|
|
166
|
+
resolution_equal: options.resolutionEqual,
|
|
167
|
+
venue: options.venue,
|
|
168
|
+
status: options.status,
|
|
169
|
+
limit: options.limit ?? 100,
|
|
170
|
+
cursor,
|
|
171
|
+
};
|
|
172
|
+
const data = await this.call(() => this.api.GET("/v1/matched_events", { params: { query } }));
|
|
173
|
+
return { items: data.data, nextCursor: data.next_cursor ?? null };
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
/** sk.fills({eventId, since}) lists fills from since to until (now by default), maker legs by default. */
|
|
177
|
+
fills(options) {
|
|
178
|
+
const to = iso(options.until) ?? new Date().toISOString();
|
|
179
|
+
return new Paginated(async (cursor) => {
|
|
180
|
+
const query = {
|
|
181
|
+
venue: options.venue,
|
|
182
|
+
market_id: options.marketId,
|
|
183
|
+
event_id: options.eventId,
|
|
184
|
+
from: iso(options.since),
|
|
185
|
+
to,
|
|
186
|
+
leg: options.leg ?? "maker",
|
|
187
|
+
limit: options.limit ?? 100,
|
|
188
|
+
cursor,
|
|
189
|
+
};
|
|
190
|
+
const data = await this.call(() => this.api.GET("/v1/fills", { params: { query } }));
|
|
191
|
+
return { items: data.data, nextCursor: data.next_cursor ?? null };
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
/** sk.candles({res, since}) lists candles of one size from since to until. */
|
|
195
|
+
candles(options) {
|
|
196
|
+
const to = iso(options.until) ?? new Date().toISOString();
|
|
197
|
+
return new Paginated(async (cursor) => {
|
|
198
|
+
const query = {
|
|
199
|
+
venue: options.venue,
|
|
200
|
+
market_id: options.marketId,
|
|
201
|
+
event_id: options.eventId,
|
|
202
|
+
from: iso(options.since),
|
|
203
|
+
to,
|
|
204
|
+
res: options.res,
|
|
205
|
+
limit: options.limit ?? 100,
|
|
206
|
+
cursor,
|
|
207
|
+
};
|
|
208
|
+
const data = await this.call(() => this.api.GET("/v1/candles", { params: { query } }));
|
|
209
|
+
return { items: data.data, nextCursor: data.next_cursor ?? null };
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
/** sk.resolutions({venue, marketId, createdSince}) lists resolution events, of up to 100 markets when marketId is given. */
|
|
213
|
+
resolutions(options = {}) {
|
|
214
|
+
return new Paginated(async (cursor) => {
|
|
215
|
+
const query = {
|
|
216
|
+
venue: options.venue,
|
|
217
|
+
market_id: options.marketId,
|
|
218
|
+
created_since: iso(options.createdSince),
|
|
219
|
+
limit: options.limit ?? 100,
|
|
220
|
+
cursor,
|
|
221
|
+
};
|
|
222
|
+
const data = await this.call(() => this.api.GET("/v1/resolutions", { params: { query } }));
|
|
223
|
+
return { items: data.data, nextCursor: data.next_cursor ?? null };
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
/** sk.fees({venue}) lists the fee schedule rows. */
|
|
227
|
+
fees(options = {}) {
|
|
228
|
+
return new Paginated(async (cursor) => {
|
|
229
|
+
const query = { venue: options.venue, limit: options.limit ?? 100, cursor };
|
|
230
|
+
const data = await this.call(() => this.api.GET("/v1/fees", { params: { query } }));
|
|
231
|
+
return { items: data.data, nextCursor: data.next_cursor ?? null };
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
/** sk.venues() lists the venues with their rights and flags. */
|
|
235
|
+
async venues() {
|
|
236
|
+
const data = await this.call(() => this.api.GET("/v1/venues"));
|
|
237
|
+
return data.data;
|
|
238
|
+
}
|
|
239
|
+
/** sk.quoteContext(eventId) reads a matched event's quote context: its active members' tokens, outcome maps and fee rows (one metered data call). */
|
|
240
|
+
quoteContext(eventId) {
|
|
241
|
+
return this.call(() => this.api.GET("/v1/quote-context/{event_id}", { params: { path: { event_id: eventId } } }));
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* sk.quote(eventId, {outcome, side, size}) is the best venue, or split
|
|
245
|
+
* across venues, for size contracts of one outcome once each venue's taker
|
|
246
|
+
* fee is counted, in micro-units: the plan and its legs, every reachable
|
|
247
|
+
* venue alone, the members left out and why, and warnings (rule_mismatch,
|
|
248
|
+
* fee_unknown, partial). One call to Skewtape for the context; the books
|
|
249
|
+
* are read from each venue with a plain fetch that never carries the key.
|
|
250
|
+
* It places nothing. A rule mismatch throws RuleMismatchError unless
|
|
251
|
+
* allowRuleMismatch; no readable book throws VenueUnreachableError.
|
|
252
|
+
*/
|
|
253
|
+
async quote(eventId, options) {
|
|
254
|
+
return quoteFromContext(await this.quoteContext(eventId), options);
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* sk.backtest(trades, {asOf}) replays trades the caller supplies and gives
|
|
258
|
+
* their fee-aware P&L (runBacktest, which this feeds). It is arithmetic on
|
|
259
|
+
* history: it forecasts nothing, suggests no trade and optimises nothing.
|
|
260
|
+
* Unless passed in, it reads the fee rows of the trades' venues, their
|
|
261
|
+
* markets and those markets' resolution events (a request per 100
|
|
262
|
+
* markets), and the history window up to each trade without a price, and
|
|
263
|
+
* up to asOf for a market its resolutions do not settle by then, per
|
|
264
|
+
* market. A candle window starts a candle earlier, since a candle prices a
|
|
265
|
+
* time once it has ended. A backtest whose trades have prices and whose
|
|
266
|
+
* markets are settled reads no history. A window with no row at or before
|
|
267
|
+
* a trade's time is a BacktestError price_missing: widen lookbackSeconds
|
|
268
|
+
* or give the price.
|
|
269
|
+
*/
|
|
270
|
+
async backtest(trades, options = {}) {
|
|
271
|
+
const history = options.history ?? "fills";
|
|
272
|
+
if (history !== "fills" && history !== "candles" && history !== "none")
|
|
273
|
+
throw new ConfigurationError(`history is fills, candles or none, not ${String(history)}`);
|
|
274
|
+
const candleRes = options.candleRes ?? "1h";
|
|
275
|
+
if (candleRes !== "1m" && candleRes !== "1h" && candleRes !== "1d")
|
|
276
|
+
throw new ConfigurationError(`candleRes is 1m, 1h or 1d, not ${String(candleRes)}`);
|
|
277
|
+
const list = [...trades];
|
|
278
|
+
const asOf = options.asOf ?? new Date();
|
|
279
|
+
const asOfMicros = parseMicros(asOf, "asOf");
|
|
280
|
+
const lookback = BigInt(Math.round((options.lookbackSeconds ?? 86_400) * 1e6));
|
|
281
|
+
const byVenue = new Map();
|
|
282
|
+
for (const t of list) {
|
|
283
|
+
const ids = byVenue.get(t.venue) ?? [];
|
|
284
|
+
if (!ids.includes(t.market_id))
|
|
285
|
+
ids.push(t.market_id);
|
|
286
|
+
byVenue.set(t.venue, ids);
|
|
287
|
+
}
|
|
288
|
+
let fees = options.fees;
|
|
289
|
+
if (fees === undefined) {
|
|
290
|
+
const rows = [];
|
|
291
|
+
for (const venue of byVenue.keys())
|
|
292
|
+
rows.push(...(await this.fees({ venue }).toArray()));
|
|
293
|
+
fees = rows;
|
|
294
|
+
}
|
|
295
|
+
let markets = options.markets ? [...options.markets] : undefined;
|
|
296
|
+
if (markets === undefined) {
|
|
297
|
+
markets = [];
|
|
298
|
+
for (const [venue, ids] of byVenue) {
|
|
299
|
+
for (let start = 0; start < ids.length; start += 100) {
|
|
300
|
+
markets.push(...(await this.markets({ venue, marketId: ids.slice(start, start + 100) }).toArray()));
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
let resolutions = options.resolutions;
|
|
305
|
+
if (resolutions === undefined) {
|
|
306
|
+
const rows = [];
|
|
307
|
+
for (const [venue, ids] of byVenue) {
|
|
308
|
+
for (let start = 0; start < ids.length; start += 100) {
|
|
309
|
+
rows.push(...(await this.resolutions({ venue, marketId: ids.slice(start, start + 100), limit: 1000 }).toArray()));
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
resolutions = rows;
|
|
313
|
+
}
|
|
314
|
+
const resolutionRows = [...resolutions];
|
|
315
|
+
let fills = options.fills;
|
|
316
|
+
let candles = options.candles;
|
|
317
|
+
if (history !== "none" && (history === "fills" ? fills : candles) === undefined) {
|
|
318
|
+
const rows = [];
|
|
319
|
+
const bucket = history === "candles" ? CANDLE_MICROS[candleRes] : 0n;
|
|
320
|
+
for (const [venue, ids] of byVenue) {
|
|
321
|
+
for (const marketId of ids) {
|
|
322
|
+
const times = list
|
|
323
|
+
.filter((t) => t.venue === venue && t.market_id === marketId && (t.price === undefined || t.price === null) && (t.notional_raw === undefined || t.notional_raw === null))
|
|
324
|
+
.map((t) => parseMicros(t.ts, "trade ts"));
|
|
325
|
+
const own = resolutionRows.filter((r) => r.venue === venue && r.market_id === marketId);
|
|
326
|
+
if (!settledAt(own, asOf))
|
|
327
|
+
times.push(asOfMicros);
|
|
328
|
+
for (const [since, until] of windows(times, lookback + bucket)) {
|
|
329
|
+
const page = history === "fills"
|
|
330
|
+
? await this.fills({ venue, marketId, since: dateOf(since), until: dateOf(until) }).toArray()
|
|
331
|
+
: await this.candles({ res: candleRes, venue, marketId, since: dateOf(since), until: dateOf(until) }).toArray();
|
|
332
|
+
rows.push(...page);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
if (history === "fills")
|
|
337
|
+
fills = rows;
|
|
338
|
+
else
|
|
339
|
+
candles = rows;
|
|
340
|
+
}
|
|
341
|
+
return runBacktest(list, {
|
|
342
|
+
asOf,
|
|
343
|
+
fees,
|
|
344
|
+
markets,
|
|
345
|
+
resolutions: resolutionRows,
|
|
346
|
+
fills: fills ?? [],
|
|
347
|
+
candles: candles ?? [],
|
|
348
|
+
prefer: history === "candles" ? "candles" : "fills",
|
|
349
|
+
...(options.voidPayout !== undefined ? { voidPayout: options.voidPayout } : {}),
|
|
350
|
+
...(options.onMissingFee !== undefined ? { onMissingFee: options.onMissingFee } : {}),
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* sk.stream({topics}) is the live feed (Builder and Pro), an async iterable
|
|
355
|
+
* of the server's messages that reconnects by itself: it resumes from the
|
|
356
|
+
* last seq it yielded, so nothing is missed or sent twice, and drops a
|
|
357
|
+
* trade it already yielded. venues, markets and events filter as the feed's
|
|
358
|
+
* subscribe does; resumeFrom starts from a seq kept earlier; maxReconnects
|
|
359
|
+
* bounds the failed connections in a row. It uses the platform's WebSocket;
|
|
360
|
+
* on Node 20 pass one (the ws package's): {WebSocket}.
|
|
361
|
+
*/
|
|
362
|
+
stream(options) {
|
|
363
|
+
// The client's own sleep only when one was given (tests): the stream's
|
|
364
|
+
// default is a timer close() clears.
|
|
365
|
+
return new Stream(feedUrl(this.baseUrl), this.#apiKey, options, this.userSleep);
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* sk.kalshi() is the optional live Kalshi client, under the user's own
|
|
369
|
+
* Kalshi membership ({accessKey, privateKey} sign its calls; without them
|
|
370
|
+
* they are public requests). It calls Kalshi's host from this machine and
|
|
371
|
+
* shares nothing with this client, not the API key: Skewtape's Kalshi data
|
|
372
|
+
* ships from sk.markets, sk.fills, sk.candles and sk.resolutions, under the
|
|
373
|
+
* Kalshi data licence; this is a reading of the live venue beside it.
|
|
374
|
+
*/
|
|
375
|
+
kalshi(options = {}) {
|
|
376
|
+
return new Kalshi(options);
|
|
377
|
+
}
|
|
378
|
+
/** sk.route: an unsigned order with your own code or fee (route.ts); it sends nothing, the API key included. */
|
|
379
|
+
route = route;
|
|
380
|
+
/** sk.geoblock: the venues' own checks (geoblock.ts); they never send the API key. */
|
|
381
|
+
geoblock = geoblock;
|
|
382
|
+
/**
|
|
383
|
+
* The History snapshot and the monthly deltas, file by file, with an API key
|
|
384
|
+
* on the Builder or Pro plan (a History purchase downloads from the
|
|
385
|
+
* dashboard; a key on Free or History is a ForbiddenPlanError naming
|
|
386
|
+
* builder):
|
|
387
|
+
* - sk.download.bundles() lists the bundles the account may download, each
|
|
388
|
+
* with its files' paths, sizes and sha256 (GET /v1/bundles);
|
|
389
|
+
* - sk.download.url(bundle, path) is a file's link, valid for an hour and
|
|
390
|
+
* logged against the account (POST /v1/downloads/{bundle}/url);
|
|
391
|
+
* - sk.download.file(bundle, path, dest) fetches that link to dest, resuming
|
|
392
|
+
* a partial download and checking the sha256;
|
|
393
|
+
* - sk.download.bundle(bundle, folder) does that for every file, skipping
|
|
394
|
+
* those already there, then checks the folder against its
|
|
395
|
+
* CHECKSUMS.sha256;
|
|
396
|
+
* - sk.download(url, dest, {sha256}) fetches a link already in hand, such as
|
|
397
|
+
* one the dashboard gave, and sk.download.verify(folder) checks a folder.
|
|
398
|
+
* A link carries no key: files are fetched with a plain fetch (options.fetch
|
|
399
|
+
* to pass one), so the file host never sees the API key.
|
|
400
|
+
*/
|
|
401
|
+
download = Object.assign((url, dest, options = {}) => downloadFile(url, dest, options), {
|
|
402
|
+
verify: (folder) => verifyChecksums(folder),
|
|
403
|
+
bundles: async () => (await this.call(() => this.api.GET("/v1/bundles"))).bundles,
|
|
404
|
+
url: (bundle, path) => this.call(() => this.api.POST("/v1/downloads/{bundle}/url", { params: { path: { bundle } }, body: { path } })),
|
|
405
|
+
file: async (bundle, path, dest, options = {}) => {
|
|
406
|
+
const sha256 = options.sha256 ?? (await this.bundleFile(bundle, path)).sha256;
|
|
407
|
+
if (await holds(dest, sha256))
|
|
408
|
+
return dest;
|
|
409
|
+
const link = await this.download.url(bundle, path);
|
|
410
|
+
return downloadFile(link.url, dest, { ...options, sha256 });
|
|
411
|
+
},
|
|
412
|
+
bundle: async (bundle, folder, options = {}) => {
|
|
413
|
+
const root = resolve(folder);
|
|
414
|
+
for (const f of (await this.findBundle(bundle)).files) {
|
|
415
|
+
const dest = resolve(root, f.path);
|
|
416
|
+
if (!dest.startsWith(root + sep))
|
|
417
|
+
throw new SkewtapeError(`${bundle} lists ${f.path}, a path outside the folder`);
|
|
418
|
+
const progress = options.onProgress;
|
|
419
|
+
await this.download.file(bundle, f.path, dest, {
|
|
420
|
+
sha256: f.sha256,
|
|
421
|
+
fetch: options.fetch,
|
|
422
|
+
onProgress: progress && ((done, total) => progress(f.path, done, total)),
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
await verifyChecksums(root);
|
|
426
|
+
return root;
|
|
427
|
+
},
|
|
428
|
+
});
|
|
429
|
+
async findBundle(bundle) {
|
|
430
|
+
const found = (await this.download.bundles()).find((b) => b.id === bundle);
|
|
431
|
+
if (!found)
|
|
432
|
+
throw new NotFoundError(`${bundle} is not among the bundles this account may download`, { status: 404 });
|
|
433
|
+
return found;
|
|
434
|
+
}
|
|
435
|
+
async bundleFile(bundle, path) {
|
|
436
|
+
const found = (await this.findBundle(bundle)).files.find((f) => f.path === path);
|
|
437
|
+
if (!found)
|
|
438
|
+
throw new NotFoundError(`${bundle} lists no file ${path}`, { status: 404 });
|
|
439
|
+
return found;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Files from a bundle: a download that resumes and checks its sha256, and the
|
|
3
|
+
* check of a whole folder against its CHECKSUMS.sha256. The History snapshot
|
|
4
|
+
* and the monthly deltas ship behind links that work for an hour: a Builder or
|
|
5
|
+
* Pro key asks for them (sk.download.file and sk.download.bundle, through POST
|
|
6
|
+
* /v1/downloads/{bundle}/url), and the dashboard hands them out too. This
|
|
7
|
+
* module takes the link itself.
|
|
8
|
+
*/
|
|
9
|
+
export interface DownloadOptions {
|
|
10
|
+
/** The expected sha256 (hex); the file is checked after the download and a wrong one is deleted. */
|
|
11
|
+
sha256?: string | undefined;
|
|
12
|
+
/** Called after each chunk with the bytes so far and the total when known. */
|
|
13
|
+
onProgress?: ((done: number, total: number | undefined) => void) | undefined;
|
|
14
|
+
/** The fetch to use (the global one by default). The file host never sees the API key. */
|
|
15
|
+
fetch?: typeof fetch | undefined;
|
|
16
|
+
}
|
|
17
|
+
/** Whether dest already holds the file whose sha256 (hex) is given. */
|
|
18
|
+
export declare function holds(dest: string, sha256: string): Promise<boolean>;
|
|
19
|
+
/**
|
|
20
|
+
* Download url to dest, resuming a partial download (dest plus .part) with a
|
|
21
|
+
* Range request, and verify sha256 when given. A dest that exists with the
|
|
22
|
+
* expected sha256 is not downloaded again; a .part that already holds the
|
|
23
|
+
* whole file (the host answers 416 to the range) goes straight to the check.
|
|
24
|
+
* Returns dest.
|
|
25
|
+
*/
|
|
26
|
+
export declare function downloadFile(url: string, dest: string, options?: DownloadOptions): Promise<string>;
|
|
27
|
+
/**
|
|
28
|
+
* Check every file a bundle's CHECKSUMS.sha256 lists ("<sha256> <path>", one
|
|
29
|
+
* per line). Returns the paths checked; throws ChecksumError on the first
|
|
30
|
+
* mismatch and SkewtapeError on a listed file that is missing or on a path
|
|
31
|
+
* that leaves the folder (absolute, or with a .. segment).
|
|
32
|
+
*/
|
|
33
|
+
export declare function verifyChecksums(folder: string, checksums?: string): Promise<string[]>;
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Files from a bundle: a download that resumes and checks its sha256, and the
|
|
3
|
+
* check of a whole folder against its CHECKSUMS.sha256. The History snapshot
|
|
4
|
+
* and the monthly deltas ship behind links that work for an hour: a Builder or
|
|
5
|
+
* Pro key asks for them (sk.download.file and sk.download.bundle, through POST
|
|
6
|
+
* /v1/downloads/{bundle}/url), and the dashboard hands them out too. This
|
|
7
|
+
* module takes the link itself.
|
|
8
|
+
*/
|
|
9
|
+
import { createHash } from "node:crypto";
|
|
10
|
+
import { createReadStream, createWriteStream } from "node:fs";
|
|
11
|
+
import { mkdir, readFile, rename, stat, unlink } from "node:fs/promises";
|
|
12
|
+
import { dirname, isAbsolute, join } from "node:path";
|
|
13
|
+
import { Readable } from "node:stream";
|
|
14
|
+
import { pipeline } from "node:stream/promises";
|
|
15
|
+
import { ChecksumError, SkewtapeError } from "./errors.js";
|
|
16
|
+
async function sha256Of(path) {
|
|
17
|
+
const hash = createHash("sha256");
|
|
18
|
+
await pipeline(createReadStream(path), hash);
|
|
19
|
+
return hash.digest("hex");
|
|
20
|
+
}
|
|
21
|
+
async function sizeOf(path) {
|
|
22
|
+
try {
|
|
23
|
+
return (await stat(path)).size;
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return 0;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
// The file's size from a 416's Content-Range header, "bytes */N".
|
|
30
|
+
function unsatisfiedTotal(resp) {
|
|
31
|
+
const range = resp.headers.get("content-range") ?? "";
|
|
32
|
+
if (!range.startsWith("bytes */"))
|
|
33
|
+
return undefined;
|
|
34
|
+
const n = Number(range.slice("bytes */".length));
|
|
35
|
+
return Number.isFinite(n) ? n : undefined;
|
|
36
|
+
}
|
|
37
|
+
function totalSize(resp, have) {
|
|
38
|
+
const range = resp.headers.get("content-range");
|
|
39
|
+
if (range && range.includes("/")) {
|
|
40
|
+
const n = Number(range.slice(range.lastIndexOf("/") + 1));
|
|
41
|
+
return Number.isFinite(n) ? n : undefined;
|
|
42
|
+
}
|
|
43
|
+
const length = resp.headers.get("content-length");
|
|
44
|
+
if (length) {
|
|
45
|
+
const n = Number(length);
|
|
46
|
+
return Number.isFinite(n) ? have + n : undefined;
|
|
47
|
+
}
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
/** Whether dest already holds the file whose sha256 (hex) is given. */
|
|
51
|
+
export async function holds(dest, sha256) {
|
|
52
|
+
return (await exists(dest)) && (await sha256Of(dest)) === sha256.toLowerCase();
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Download url to dest, resuming a partial download (dest plus .part) with a
|
|
56
|
+
* Range request, and verify sha256 when given. A dest that exists with the
|
|
57
|
+
* expected sha256 is not downloaded again; a .part that already holds the
|
|
58
|
+
* whole file (the host answers 416 to the range) goes straight to the check.
|
|
59
|
+
* Returns dest.
|
|
60
|
+
*/
|
|
61
|
+
export async function downloadFile(url, dest, options = {}) {
|
|
62
|
+
const fetchFn = options.fetch ?? fetch;
|
|
63
|
+
const expected = options.sha256?.toLowerCase();
|
|
64
|
+
if (expected && (await sizeOf(dest)) > 0 && (await sha256Of(dest)) === expected)
|
|
65
|
+
return dest;
|
|
66
|
+
await mkdir(dirname(dest), { recursive: true });
|
|
67
|
+
const part = `${dest}.part`;
|
|
68
|
+
let have = await sizeOf(part);
|
|
69
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
70
|
+
const resp = await fetchFn(url, { headers: have ? { Range: `bytes=${have}-` } : {} });
|
|
71
|
+
if (have && resp.status === 416) {
|
|
72
|
+
// 416 on a resume: the .part is whole when the file's size (Content-Range: bytes */N)
|
|
73
|
+
// is what it holds; otherwise it is longer than the file or another file's, so start over.
|
|
74
|
+
await resp.body?.cancel();
|
|
75
|
+
if (unsatisfiedTotal(resp) === have)
|
|
76
|
+
break;
|
|
77
|
+
await unlink(part).catch(() => undefined);
|
|
78
|
+
have = 0;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
if (have && resp.status === 200)
|
|
82
|
+
have = 0; // the server ignored the range: start over
|
|
83
|
+
else if (have && resp.status !== 206)
|
|
84
|
+
throw new SkewtapeError(`download ${url}: HTTP ${resp.status} on resume`, { status: resp.status });
|
|
85
|
+
else if (!have && resp.status !== 200)
|
|
86
|
+
throw new SkewtapeError(`download ${url}: HTTP ${resp.status}`, { status: resp.status });
|
|
87
|
+
if (!resp.body)
|
|
88
|
+
throw new SkewtapeError(`download ${url}: no body`, { status: resp.status });
|
|
89
|
+
const total = totalSize(resp, have);
|
|
90
|
+
let done = have;
|
|
91
|
+
const progress = options.onProgress;
|
|
92
|
+
const counted = Readable.from(countChunks(resp.body, (n) => {
|
|
93
|
+
done += n;
|
|
94
|
+
progress?.(done, total);
|
|
95
|
+
}));
|
|
96
|
+
await pipeline(counted, createWriteStream(part, { flags: have ? "a" : "w" }));
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
if (expected) {
|
|
100
|
+
const actual = await sha256Of(part);
|
|
101
|
+
if (actual !== expected) {
|
|
102
|
+
await unlink(part).catch(() => undefined);
|
|
103
|
+
throw new ChecksumError(dest, expected, actual);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
await rename(part, dest);
|
|
107
|
+
return dest;
|
|
108
|
+
}
|
|
109
|
+
async function* countChunks(body, onChunk) {
|
|
110
|
+
const reader = body.getReader();
|
|
111
|
+
for (;;) {
|
|
112
|
+
const { done, value } = await reader.read();
|
|
113
|
+
if (done)
|
|
114
|
+
return;
|
|
115
|
+
onChunk(value.byteLength);
|
|
116
|
+
yield value;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Check every file a bundle's CHECKSUMS.sha256 lists ("<sha256> <path>", one
|
|
121
|
+
* per line). Returns the paths checked; throws ChecksumError on the first
|
|
122
|
+
* mismatch and SkewtapeError on a listed file that is missing or on a path
|
|
123
|
+
* that leaves the folder (absolute, or with a .. segment).
|
|
124
|
+
*/
|
|
125
|
+
export async function verifyChecksums(folder, checksums = "CHECKSUMS.sha256") {
|
|
126
|
+
const text = await readFile(join(folder, checksums), "utf8");
|
|
127
|
+
const checked = [];
|
|
128
|
+
for (const raw of text.split("\n")) {
|
|
129
|
+
const line = raw.trim();
|
|
130
|
+
if (!line || line.startsWith("#"))
|
|
131
|
+
continue;
|
|
132
|
+
const space = line.indexOf(" ");
|
|
133
|
+
const digest = line.slice(0, space).toLowerCase();
|
|
134
|
+
const rel = line.slice(space + 1).trim().replace(/^\*/, "");
|
|
135
|
+
if (isAbsolute(rel) || rel.split(/[\\/]+/).includes(".."))
|
|
136
|
+
throw new SkewtapeError(`${checksums} names ${rel}, a path outside the folder`);
|
|
137
|
+
const path = join(folder, rel);
|
|
138
|
+
if ((await sizeOf(path)) === 0 && !(await exists(path)))
|
|
139
|
+
throw new SkewtapeError(`${path}: listed in ${checksums} and missing`);
|
|
140
|
+
const actual = await sha256Of(path);
|
|
141
|
+
if (actual !== digest)
|
|
142
|
+
throw new ChecksumError(path, digest, actual);
|
|
143
|
+
checked.push(path);
|
|
144
|
+
}
|
|
145
|
+
return checked;
|
|
146
|
+
}
|
|
147
|
+
async function exists(path) {
|
|
148
|
+
try {
|
|
149
|
+
await stat(path);
|
|
150
|
+
return true;
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
}
|