briskapi 0.3.0__tar.gz → 0.3.1__tar.gz

{briskapi-0.3.0 → briskapi-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: briskapi
-Version: 0.3.0
+Version: 0.3.1
 Summary: Unofficial Python API, live feed and shared archive for BRiSK auction data
 License-Expression: MIT
 Project-URL: Source, https://github.com/honvl/BRiSKapi

{briskapi-0.3.0 → briskapi-0.3.1}/briskapi/__init__.py

@@ -18,7 +18,7 @@ from briskapi._live import Feed, connect as _connect, record as _record, stream
 from briskapi._market import Market, Ticker
 from briskapi._recording import JST, BriskError, NotFoundError, Recording, Table
 
-__version__ = '0.3.0'
+__version__ = '0.3.1'
 __all__ = ['JST', 'Archive', 'BriskError', 'Feed', 'Market', 'NotFoundError', 'Recording', 'Table', 'Ticker',
            'connect', 'consent', 'current', 'load', 'pull', 'record', 'recordings', 'stream']
 

{briskapi-0.3.0 → briskapi-0.3.1}/briskapi/decoder/decoder.cjs

@@ -52,10 +52,37 @@ function loadAssets(cache) {
   return assets;
 }
 
+// Position of the stock view's fields, relative to a version's base. Protocol 16000 (the 2021
+// demo, and the vendor client's own parse of it) puts a header word first. 18000 (SBI's current
+// build, as an independent parser of it reads the same view) drops that word, so every index is
+// one lower. The fields and their order are otherwise identical. Any other version is refused
+// rather than read at a guessed offset.
+const STOCK_VIEW_SHIFT = { 16000: 0, 18000: -1 };
+
+function stockViewShift(protocolVersion) {
+  if (!(protocolVersion in STOCK_VIEW_SHIFT)) {
+    throw new Error(`Stock view layout for protocol ${protocolVersion} is not known (known: ${Object.keys(STOCK_VIEW_SHIFT).join(', ')})`);
+  }
+  return STOCK_VIEW_SHIFT[protocolVersion];
+}
+
+// `words` is the raw _getStockView buffer; `ohlc` the number of OHLC bars ahead of the quote fields.
+function parseStockView(words, ohlc, shift) {
+  const a = words.subarray(6 * ohlc + shift);
+  return { frame: a[19], max_frame: a[20], source_time_us: u64(a, 17), last_price10: words[1 + shift],
+    open_price10: words[2 + shift], bid_price10: a[7], ask_price10: a[8], indicative_price10: a[9],
+    indicative_volume: u64(a, 10), indicative_side: a[12], closing_indicative_price10: a[13],
+    closing_indicative_volume: u64(a, 14), quote_flag: a[21], quote_side: a[22],
+    special_quote_time_us: u64(a, 23), indicative_open_price10: a[32], auction_reference_price10: a[35],
+    volume: u64(a, 3) };
+}
+
 class Decoder {
   // options.protocolVersion: 16000 for the Next demo (default), 18000 for SBI BRiSK.
-  // options.callbacks: { send(Buffer), heartbeat(ns BigInt), frameNumbers(Array), basePrice(n),
+  // options.callbacks: { send(Buffer), heartbeat(ns BigInt), frameNumbers(Array), basePrice(issueIndex),
   //   authError(), marketFinished() }, the vendor client's six WASM callbacks.
+  // options.boot: { exceptional_sq, base_prices } from the app boot response, applied in the
+  //   vendor client's order: after the master is loaded, before the snapshot.
   static async create(assets, options = {}) {
     // Isolate the legacy glue's globals and process exception handlers. No UI,
     // network, filesystem access or account state is needed inside this VM.
@@ -74,8 +101,10 @@ class Decoder {
     return new Decoder(wasm, assets, options);
   }
 
-  constructor(w, assets, { protocolVersion = manifest.protocol_version, callbacks = {} } = {}) {
+  constructor(w, assets, { protocolVersion = manifest.protocol_version, callbacks = {}, boot = {} } = {}) {
     this.w = w;
+    this.layoutShift = stockViewShift(protocolVersion);
+    this.basePriceQueue = new Set();
     this.authError = false;
     this.marketFinished = false;
     this.initialFrames = null;
@@ -96,7 +125,8 @@ class Decoder {
         this.lastHeartbeat = BigInt(low >>> 0) + (BigInt(high >>> 0) << 32n);
         callbacks.heartbeat?.(this.lastHeartbeat);
       }, 'viii'),
-      add((id, price) => callbacks.basePrice?.(price), 'vii'),
+      // basePrice(id, issueIndex): the decoder changed an issue's base price or limits.
+      add((id, issueIndex) => { this.basePriceQueue.add(issueIndex); callbacks.basePrice?.(issueIndex); }, 'vii'),
       add(() => { this.authError = true; callbacks.authError?.(); }, 'vii'),
       add(() => { this.marketFinished = true; callbacks.marketFinished?.(); }, 'vi'),
       protocolVersion);
@@ -114,10 +144,54 @@ class Decoder {
         name: Buffer.from(w.HEAPU8.subarray(this.buf + 144 * issue_id + 68,
           this.buf + 144 * issue_id + 68 + a[16])).toString('utf8') };
     });
+    this.boot = this.applyBoot(boot);
     this.unserialize(assets['snapshot.dat']);
     this.date = String(w._getDate(this.id));
   }
 
+  // The vendor client pushes the boot response's exceptional special quotes and base-price
+  // changes into the decoder right after the master, then re-reads the master's limits.
+  applyBoot({ exceptional_sq = [], base_prices = [] } = {}) {
+    const byCode = new Map(this.master.map(m => [m.code, m.issue_id]));
+    const number = (value, what) => {
+      if (!Number.isFinite(value)) throw new Error(`Boot ${what} is not a number: ${JSON.stringify(value)}`);
+      return value;
+    };
+    for (const sq of exceptional_sq) {
+      if (!/^\d+$/.test(String(sq.issue_code))) {
+        throw new Error(`Boot exceptional_sq names issue code ${sq.issue_code}; only numeric codes are supported`);
+      }
+      this.w._pushSQ(this.id, Number(sq.issue_code), sq.buy_sell === 'B' ? 0 : 1, 10 * number(sq.jump_range, 'jump_range'),
+        number(sq.jump_sec, 'jump_sec'), 10 * number(sq.quote_limit_down, 'quote_limit_down'),
+        10 * number(sq.quote_limit_up, 'quote_limit_up'));
+    }
+    for (const change of base_prices) {
+      const issue = byCode.get(String(change.issue_code));
+      if (issue === undefined) throw new Error(`Boot base_prices names issue code ${change.issue_code}, which is not in the master`);
+      this.w._pushBasePrice(this.id, issue, number(change.base_price10, 'base_price10'),
+        number(change.limit_down10, 'limit_down10'), number(change.limit_up10, 'limit_up10'));
+      this.basePriceQueue.add(issue);
+    }
+    return { exceptional_sq: exceptional_sq.length, base_prices: base_prices.length, applied: this.applyBasePrice() };
+  }
+
+  // Re-read the master's price limits for issues whose base price changed (the vendor client
+  // does this after boot and after every frame); cheap when nothing is queued.
+  applyBasePrice() {
+    if (!this.basePriceQueue.size) return 0;
+    this.w._getStockMaster(this.id, this.buf, this.master.length * 144);
+    let applied = 0;
+    for (const issue of this.basePriceQueue) {
+      const m = this.master[issue];
+      if (!m) throw new Error(`Base price change for unknown issue index ${issue}`);
+      const a = new Uint32Array(this.w.HEAPU8.buffer, this.buf + 144 * issue, 36);
+      m.base_price10 = a[2]; m.limit_up10 = a[3]; m.limit_down10 = a[4];
+      applied++;
+    }
+    this.basePriceQueue.clear();
+    return applied;
+  }
+
   push(data) {
     for (let offset = 0; offset < data.length; offset += BUFFER) {
       const part = data.subarray(offset, offset + BUFFER);
@@ -214,15 +288,7 @@ class Decoder {
     if (!m) throw new Error(`Unknown issue ID: ${issue_id}`);
     if (!this.w._getStockView(this.id, issue_id, this.buf)) throw new Error(`Stock view unavailable: ${m.code}`);
     const words = new Uint32Array(this.w.HEAPU8.buffer, this.buf, this.viewWords);
-    const a = words.subarray(6 * this.ohlc);
-    const quote = { issue_id, code: m.code, frame: a[19], max_frame: a[20],
-      source_time_us: u64(a, 17), last_price10: words[1], open_price10: words[2],
-      bid_price10: a[7], ask_price10: a[8], indicative_price10: a[9],
-      indicative_volume: u64(a, 10), indicative_side: a[12],
-      closing_indicative_price10: a[13], closing_indicative_volume: u64(a, 14),
-      quote_flag: a[21], quote_side: a[22], special_quote_time_us: u64(a, 23),
-      indicative_open_price10: a[32], auction_reference_price10: a[35],
-      volume: u64(a, 3) };
+    const quote = { issue_id, code: m.code, ...parseStockView(words, this.ohlc, this.layoutShift) };
     // SBI's build has no portfolio export, so its quotes carry no issue_status.
     if (this.w._getPortfolio) {
       if (!this.w._getPortfolio(this.id, Number(m.code), this.buf)) throw new Error(`Portfolio unavailable: ${m.code}`);
@@ -315,7 +381,7 @@ async function main(argv) {
     } });
 }
 
-module.exports = { Decoder, QuoteHistory, frames, u64, loadAssets, replay, main };
+module.exports = { Decoder, QuoteHistory, frames, u64, loadAssets, replay, main, parseStockView, stockViewShift, STOCK_VIEW_SHIFT };
 if (require.main === module) main(process.argv.slice(2)).catch(error => {
   console.error(error.message); process.exitCode = 1;
 });

{briskapi-0.3.0 → briskapi-0.3.1}/briskapi/decoder/engineio.cjs

@@ -248,14 +248,38 @@ class Link {
 
 // The vendor client closes the connection when its last heartbeat is older than 7 s, but
 // only once it has seen one. Beats come from the WASM's heartbeat callback.
-function heartbeatMonitor({ intervalMs = 7000, checkEveryMs = 1000, onFail, now = () => performance.now() }) {
+function heartbeatMonitor({ intervalMs = 7000, checkEveryMs = 1000, onFail, now = () => performance.now(), staleCheck = null }) {
   let last = null;
   const timer = setInterval(() => {
     if (last !== null && now() - last > intervalMs) {
       onFail(new Error(`SBI BRiSK connection check failure: no heartbeat for ${Math.round((now() - last) / 1000)}s`));
     }
+    const stale = staleCheck?.();
+    if (stale) onFail(stale);
   }, checkEveryMs);
   return { beat() { last = now(); }, stop() { clearInterval(timer); } };
 }
 
-module.exports = { Link, SBI_PROFILE, resolveProfile, heartbeatMonitor, redact };
+// The vendor client's second liveness rule: during trading hours (08:00-15:00 JST, not the
+// 11:30-12:10 break) the decoder's data time must not lag the server clock by more than 90 s.
+// The server clock comes from the heartbeat; nothing is checked before the first one, or on a
+// day other than the market date.
+function staleDataCheck({ date, thresholdUs = 90_000_000, now = Date.now }) {
+  let server = null;
+  return {
+    beat(ns) { server = { serverMs: Number(ns / 1_000_000n), localMs: now() }; },
+    check(decoderTimeUs) {
+      if (!server || !decoderTimeUs) return null;
+      const jst = new Date(server.serverMs + (now() - server.localMs) + 9 * 3600e3);
+      if (jst.toISOString().slice(0, 10) !== date) return null;
+      const hour = jst.getUTCHours(), minute = jst.getUTCMinutes();
+      if (hour >= 15 || hour <= 7 || (hour === 11 && minute >= 30) || (hour === 12 && minute < 10)) return null;
+      const lag = 1e6 * (60 * (60 * hour + minute) + jst.getUTCSeconds()) - decoderTimeUs;
+      return lag > thresholdUs
+        ? new Error(`SBI BRiSK data is stale: the decoder's clock is ${Math.round(lag / 1e6)}s behind the server (limit ${thresholdUs / 1e6}s)`)
+        : null;
+    },
+  };
+}
+
+module.exports = { Link, SBI_PROFILE, resolveProfile, heartbeatMonitor, staleDataCheck, redact };

{briskapi-0.3.0 → briskapi-0.3.1}/briskapi/decoder/sbi.cjs

@@ -13,7 +13,7 @@
 const crypto = require('node:crypto');
 const { once } = require('node:events');
 const { Decoder } = require('./decoder.cjs');
-const { Link, resolveProfile, heartbeatMonitor, redact } = require('./engineio.cjs');
+const { Link, resolveProfile, heartbeatMonitor, staleDataCheck, redact } = require('./engineio.cjs');
 
 const ORIGIN = 'https://sbi.brisk.jp';
 const PROTOCOL_VERSION = 18000;
@@ -113,21 +113,51 @@ async function decoderAssets(session) {
   throw new Error('SBI BRiSK decoder not found in the app bundles; the site layout may have changed');
 }
 
+// The master's record layout also depends on the protocol version, and SBI's is not independently
+// confirmed. A wrong layout would read tick types as codes, limits as lot sizes and so on, so
+// refuse a master that does not look like a market. (The 2021 demo's passes every test at 99.98%.)
+function checkMaster(master, protocolVersion) {
+  const fraction = predicate => master.filter(predicate).length / (master.length || 1);
+  const problems = [];
+  if (master.length < 1000) problems.push(`only ${master.length} issues`);
+  const rules = [
+    ['numeric codes', 0.95, m => /^\d{3,4}$/.test(m.code)],
+    ['positive base prices', 0.98, m => m.base_price10 > 0],
+    ['limits around the base price', 0.98, m => m.limit_up10 > m.base_price10 && m.base_price10 > m.limit_down10],
+    ['lot sizes of 1 to 1,000,000', 0.98, m => m.lot_size >= 1 && m.lot_size <= 1e6],
+    ['names', 0.95, m => m.name.length > 0],
+  ];
+  for (const [what, minimum, predicate] of rules) {
+    const seen = fraction(predicate);
+    if (seen < minimum) problems.push(`${what}: ${(100 * seen).toFixed(1)}% (need ${(100 * minimum).toFixed(0)}%)`);
+  }
+  const distinct = new Set(master.map(m => m.code)).size / (master.length || 1);
+  if (distinct < 0.99) problems.push(`distinct codes: ${(100 * distinct).toFixed(1)}% (need 99%)`);
+  if (problems.length) {
+    throw new Error(`SBI BRiSK stock master layout not recognised for protocol ${protocolVersion}: ${problems.join('; ')}`);
+  }
+}
+
 function checkAbi(wasm) {
   const missing = REQUIRED.filter(name => typeof wasm[name] !== 'function');
   if (missing.length) throw new Error(`SBI BRiSK decoder changed; missing ${missing.join(', ')}`);
 }
 
 async function live({ cookies, codes = [], emit, fetchImpl = fetch, WebSocketImpl = WebSocket, startTimeoutMs = 60000,
-  protocolVersion = PROTOCOL_VERSION, profile: overrides = {}, trace = null, catchUp = null, heartbeatMs = 7000,
-  connectTimeoutMs = 15000, catchUpBackoffMs }) {
+  protocolVersion, profile: overrides = {}, trace = null, catchUp = null, heartbeatMs = 7000,
+  staleSeconds = 90, connectTimeoutMs = 15000, catchUpBackoffMs }) {
   const profile = resolveProfile(overrides);
   const session = new Session(cookies, fetchImpl);
   const began = performance.now();
   const frontend = await session.get('/api/frontend/boot');
   session.token = frontend.api_token;
   const app = await session.get('/api/app/boot');
-  trace?.(`boot: date ${app.date}, series ${app.series}, ws_url ${redact(app.ws_url)}`);
+  // The server says which decoder protocol it speaks (the vendor client passes it to the WASM and
+  // falls back to 16000); an explicit option wins, then the boot response, then SBI's 18000.
+  protocolVersion ??= app.flex_version ?? PROTOCOL_VERSION;
+  trace?.(`boot: date ${app.date}, series ${app.series}, flex_version ${protocolVersion}, `
+    + `${(app.base_prices || []).length} base prices, ${(app.exceptional_sq || []).length} exceptional special quotes, `
+    + `ws_url ${redact(app.ws_url)}`);
   const { assets, paths } = await decoderAssets(session);
   assets['master.dat'] = await session.get(`/api/master/${encodeURIComponent(app.master)}`, 'bytes');
   assets['snapshot.dat'] = await session.get(`/api/snapshot/${encodeURIComponent(app.snapshot)}`, 'bytes');
@@ -141,18 +171,22 @@ async function live({ cookies, codes = [], emit, fetchImpl = fetch, WebSocketImp
     settle({ code: -1, reason: '' });
   };
   let monitor = null, timer = null;  // started with the link and always stopped, so a failure cannot leave the process hanging
-  const decoder = await Decoder.create(assets, { protocolVersion, check: checkAbi, callbacks: {
+  const stale = staleSeconds ? staleDataCheck({ date: app.date, thresholdUs: staleSeconds * 1e6 }) : null;
+  const decoder = await Decoder.create(assets, { protocolVersion, check: checkAbi,
+    boot: { exceptional_sq: app.exceptional_sq, base_prices: app.base_prices }, callbacks: {
     send: bytes => { if (link) link.send(bytes); },
-    heartbeat: () => monitor?.beat(),
+    heartbeat: ns => { monitor?.beat(); stale?.beat(ns); },
     authError: () => fail(new Error('SBI BRiSK reports another WebSocket session for this user (only one is allowed); close the other one')),
     marketFinished: () => { marketFinished = true; },
   } });
+  checkMaster(decoder.master, protocolVersion);
   const selected = new Set(codes);
   const master = decoder.master.filter(m => !selected.size || selected.has(m.code));
   const missing = codes.filter(code => !master.some(m => m.code === code));
   if (missing.length) throw new Error(`Codes not in the SBI master: ${missing.join(',')}`);
   const ids = new Set(master.map(m => m.issue_id));
   const input_transport = { kind: 'sbi_websocket', origin: ORIGIN + '/', series: app.series,
+    protocol_version: protocolVersion, boot_applied: decoder.boot,
     decoder: paths, decoder_sha256: crypto.createHash('sha256').update(assets['fita.wasm']).digest('hex'),
     setup_ms: performance.now() - began };
 
@@ -209,6 +243,7 @@ async function live({ cookies, codes = [], emit, fetchImpl = fetch, WebSocketImp
   const handle = async (frame, received_unix_ms) => {
     const t0 = process.hrtime.bigint();
     decoder.feed(frame);
+    decoder.applyBasePrice();
     const quotes = decoder.changed().filter(id => ids.has(id)).map(id => decoder.quote(id));
     updates += quotes.length;
     await emit({ type: 'quotes', seq: seq++, source_time_us: decoder.time(), received_unix_ms,
@@ -224,12 +259,14 @@ async function live({ cookies, codes = [], emit, fetchImpl = fetch, WebSocketImp
     }
     try {
       decoder.feed(frame);
+      decoder.applyBasePrice();
       if (!starting && decoder.initialFrames) { starting = true; begin(); }
     } catch (error) { fail(error); }
   };
 
   try {
-    monitor = heartbeatMonitor({ intervalMs: heartbeatMs, checkEveryMs: Math.min(1000, heartbeatMs / 4), onFail: fail });
+    monitor = heartbeatMonitor({ intervalMs: heartbeatMs, checkEveryMs: Math.min(1000, heartbeatMs / 4), onFail: fail,
+      staleCheck: stale && (() => stale.check(decoder.time())) });
     timer = setTimeout(() => fail(new Error(`SBI BRiSK stream did not initialize within ${startTimeoutMs / 1000}s; `
       + 'no first frame numbers arrived (run with --trace-protocol to see what the server sent)')), startTimeoutMs);
     link = new Link({ WebSocketImpl, url: new URL(app.ws_url, ORIGIN.replace('https:', 'wss:')),
@@ -281,7 +318,7 @@ async function main(argv, env = process.env) {
     } });
 }
 
-module.exports = { Session, decoderAssets, checkAbi, live, main, fetchCatchUp, CATCH_UP_FORMATS, PROTOCOL_VERSION, REQUIRED };
+module.exports = { Session, decoderAssets, checkAbi, checkMaster, live, main, fetchCatchUp, CATCH_UP_FORMATS, PROTOCOL_VERSION, REQUIRED };
 if (require.main === module) main(process.argv.slice(2)).catch(error => {
   console.error(error.message); process.exitCode = 1;
 });

{briskapi-0.3.0 → briskapi-0.3.1}/briskapi.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: briskapi
-Version: 0.3.0
+Version: 0.3.1
 Summary: Unofficial Python API, live feed and shared archive for BRiSK auction data
 License-Expression: MIT
 Project-URL: Source, https://github.com/honvl/BRiSKapi

{briskapi-0.3.0 → briskapi-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "briskapi"
-version = "0.3.0"
+version = "0.3.1"
 description = "Unofficial Python API, live feed and shared archive for BRiSK auction data"
 readme = "README.md"
 license = "MIT"