polaris-data 0.1.0

package/dist/index.js

@@ -0,0 +1,717 @@
+// src/client.ts
+import { readFile, writeFile, mkdir as mkdir2 } from "fs/promises";
+import { dirname as dirname2 } from "path";
+import { decompress } from "fzstd";
+
+// src/errors.ts
+var PolarisError = class extends Error {
+  name = "PolarisError";
+  /** HTTP status code, if the error originated from an API response. */
+  statusCode;
+  /** Raw response body string. */
+  body;
+  constructor(message, statusCode, body) {
+    super(message);
+    this.statusCode = statusCode;
+    this.body = body;
+  }
+  toString() {
+    return this.statusCode != null ? `${this.message} (status=${this.statusCode})` : this.message;
+  }
+};
+var UnauthorizedError = class extends PolarisError {
+  name = "UnauthorizedError";
+};
+var NotFoundError = class extends PolarisError {
+  name = "NotFoundError";
+};
+var RateLimitedError = class extends PolarisError {
+  name = "RateLimitedError";
+  /** ISO 8601 timestamp (or epoch) indicating when the rate limit resets. */
+  resetAt;
+  constructor(message, statusCode, body, resetAt) {
+    super(message, statusCode, body);
+    this.resetAt = resetAt;
+  }
+};
+var StreamDecodeError = class extends PolarisError {
+  name = "StreamDecodeError";
+};
+var DownloadNotAllowedError = class extends PolarisError {
+  name = "DownloadNotAllowedError";
+};
+
+// src/utils.ts
+var EPOCH_RE = /^\d+$/;
+function toIso8601(value) {
+  if (typeof value === "string") {
+    if (EPOCH_RE.test(value) && value.length >= 13) return epochUsToIso(Number(value));
+    return value;
+  }
+  if (value instanceof Date) return dateToIso(value);
+  return value > 1e12 ? epochUsToIso(value) : epochMsToIso(value);
+}
+function toEpochUs(value) {
+  if (typeof value === "number") return value > 1e12 ? value : value * 1e3;
+  if (typeof value === "string") {
+    if (EPOCH_RE.test(value) && value.length >= 13) return Number(value);
+    return Math.round(new Date(value).getTime() * 1e3);
+  }
+  return Math.round(value.getTime() * 1e3);
+}
+function datesInRange(fromUs, toUs) {
+  const dates = [];
+  const start = new Date(fromUs / 1e3);
+  const end = new Date(toUs / 1e3);
+  const cur = new Date(
+    Date.UTC(start.getUTCFullYear(), start.getUTCMonth(), start.getUTCDate())
+  );
+  const last = new Date(
+    Date.UTC(end.getUTCFullYear(), end.getUTCMonth(), end.getUTCDate())
+  );
+  while (cur <= last) {
+    dates.push(formatUtcDate(cur));
+    cur.setUTCDate(cur.getUTCDate() + 1);
+  }
+  return dates;
+}
+function epochUsToIso(us) {
+  return epochMsToIso(us / 1e3);
+}
+function epochMsToIso(ms) {
+  return new Date(ms).toISOString().replace(/\.000Z$/, "Z");
+}
+function dateToIso(d) {
+  return d.toISOString().replace(/\.000Z$/, "Z");
+}
+function formatUtcDate(d) {
+  const y = d.getUTCFullYear();
+  const m = String(d.getUTCMonth() + 1).padStart(2, "0");
+  const day = String(d.getUTCDate()).padStart(2, "0");
+  return `${y}-${m}-${day}`;
+}
+
+// src/storage.ts
+import { mkdir, stat, link, copyFile } from "fs/promises";
+import { join, dirname } from "path";
+import { homedir, platform } from "os";
+function resolveRoot(explicit) {
+  if (explicit) return explicit;
+  if (process.env.POLARIS_ROOT) return process.env.POLARIS_ROOT;
+  if (process.env.POLARIS_DATASET_DOWNLOAD_DIR)
+    return process.env.POLARIS_DATASET_DOWNLOAD_DIR;
+  return defaultRoot();
+}
+function defaultRoot() {
+  switch (platform()) {
+    case "darwin":
+      return join(
+        homedir(),
+        "Library",
+        "Application Support",
+        "polaris"
+      );
+    case "win32":
+      return join(
+        process.env.APPDATA || join(homedir(), "AppData", "Roaming"),
+        "polaris"
+      );
+    default:
+      return process.env.XDG_DATA_HOME ? join(process.env.XDG_DATA_HOME, "polaris") : join(homedir(), ".local", "share", "polaris");
+  }
+}
+async function ensureLayout(root) {
+  const dataDir = join(root, "data");
+  const dailyDir = join(root, "daily");
+  const tmpDir = join(root, "tmp");
+  const cacheDir = join(root, "cache");
+  await Promise.all([
+    mkdir(dataDir, { recursive: true }),
+    mkdir(dailyDir, { recursive: true }),
+    mkdir(tmpDir, { recursive: true }),
+    mkdir(cacheDir, { recursive: true })
+  ]);
+  return { root, dataDir, dailyDir, tmpDir, cacheDir };
+}
+function dataFilePath(dataDir, key) {
+  return join(dataDir, key);
+}
+function dailyFilePath(dailyDir, source, market, date) {
+  return join(dailyDir, source, market, `${date}.jsonl.zst`);
+}
+async function linkOrCopy(src, dest) {
+  await mkdir(dirname(dest), { recursive: true });
+  try {
+    await link(src, dest);
+  } catch {
+    await copyFile(src, dest);
+  }
+}
+async function fileExists(path) {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// src/aggregator.ts
+function intervalToUs(interval) {
+  const m = interval.match(/^(\d+)(ms|s|m|h)$/);
+  if (!m) throw new Error(`Invalid OHLCV interval: ${interval}`);
+  const n = parseInt(m[1], 10);
+  switch (m[2]) {
+    case "ms":
+      return n * 1e3;
+    case "s":
+      return n * 1e6;
+    case "m":
+      return n * 6e7;
+    case "h":
+      return n * 36e8;
+    default:
+      throw new Error(`Unreachable`);
+  }
+}
+var OhlcvAggregator = class {
+  _intervalUs;
+  _bars = /* @__PURE__ */ new Map();
+  constructor(interval) {
+    this._intervalUs = intervalToUs(interval);
+  }
+  /** Ingest a single trade event. */
+  add(timestamp, price, quantity) {
+    const bucketTs = Math.floor(timestamp / this._intervalUs) * this._intervalUs;
+    let b = this._bars.get(bucketTs);
+    if (!b) {
+      b = {
+        ts: bucketTs,
+        open: price,
+        high: price,
+        low: price,
+        close: price,
+        volScaled: 0,
+        trades: 0
+      };
+      this._bars.set(bucketTs, b);
+    }
+    if (price > b.high) b.high = price;
+    if (price < b.low) b.low = price;
+    b.close = price;
+    b.volScaled += Math.round(quantity * 1e12);
+    b.trades += 1;
+  }
+  /**
+   * Finalise the aggregation and return sorted bars.
+   *
+   * The `open` of each subsequent bar is overwritten with the `close` of the
+   * previous bar, matching the convention used by the Python SDK.
+   */
+  finish() {
+    const bars = Array.from(this._bars.values()).sort(
+      (a, b) => a.ts - b.ts
+    );
+    for (let i = 1; i < bars.length; i++) {
+      bars[i].open = bars[i - 1].close;
+    }
+    return bars.map((b) => ({
+      timestamp: b.ts,
+      open: b.open,
+      high: b.high,
+      low: b.low,
+      close: b.close,
+      volume: b.volScaled / 1e12,
+      trades: b.trades
+    }));
+  }
+};
+
+// src/client.ts
+var VERSION = "0.1.0";
+var PolarisClient = class {
+  _apiKey;
+  _baseUrl;
+  _timeout;
+  _fetch;
+  _root;
+  _layout;
+  constructor(options = {}) {
+    this._apiKey = options.apiKey ?? readEnvApiKey();
+    this._baseUrl = new URL(options.baseUrl ?? "https://api.polaris.supply");
+    this._timeout = options.timeout ?? 3e4;
+    this._fetch = options.fetch ?? globalThis.fetch;
+    this._root = resolveRoot(options.datasetRoot);
+  }
+  // -----------------------------------------------------------------------
+  // Discovery
+  // -----------------------------------------------------------------------
+  /** Check API availability. */
+  async health() {
+    return this._getJson("/health", { auth: "none" });
+  }
+  /**
+   * Browse supported sources and markets.
+   *
+   * If neither `source` nor `market` is provided, returns the full catalog.
+   * `market` requires `source`.
+   */
+  async catalog(options = {}) {
+    const params = {};
+    if (options.source) params.source = options.source;
+    if (options.market) params.market = options.market;
+    return this._getJson("/catalog", { params, auth: "if-available" });
+  }
+  /**
+   * List available snapshot files for a source and market over a time range.
+   * Auto-paginates to return **all** matching entries.
+   */
+  async listSnapshots(options) {
+    const entries = [];
+    let cursor;
+    do {
+      const params = buildSnapshotParams(options);
+      if (cursor) params.cursor = cursor;
+      const res = await this._getJson("/snapshots", { params, auth: "if-available" });
+      entries.push(...res.snapshots);
+      cursor = res.has_more ? res.next_cursor ?? void 0 : void 0;
+    } while (cursor);
+    return entries;
+  }
+  // -----------------------------------------------------------------------
+  // Historical data – snapshot-first
+  // -----------------------------------------------------------------------
+  /**
+   * Return all standardised historical events for a time range.
+   *
+   * Reads from locally-cached daily `.jsonl.zst` snapshot files.
+   * Missing daily artifacts are discovered via `GET /snapshots` and
+   * downloaded automatically.
+   */
+  async events(options) {
+    const fromUs = toEpochUs(options.from);
+    const toUs = toEpochUs(options.to);
+    const result = [];
+    for await (const event of this._readDailyEvents(
+      options.source,
+      options.market,
+      fromUs,
+      toUs
+    )) {
+      result.push(event);
+    }
+    return result;
+  }
+  /**
+   * Return all standardised trade events for a time range.
+   *
+   * Reads from locally-cached daily snapshot files, filtering to
+   * `type === "trade"`.
+   */
+  async trades(options) {
+    const fromUs = toEpochUs(options.from);
+    const toUs = toEpochUs(options.to);
+    const result = [];
+    for await (const event of this._readDailyEvents(
+      options.source,
+      options.market,
+      fromUs,
+      toUs,
+      (e) => e.type === "trade"
+    )) {
+      result.push(event);
+    }
+    return result;
+  }
+  /**
+   * Aggregate OHLCV bars from standardised trade data.
+   *
+   * Reads from locally-cached daily snapshot files and aggregates in memory
+   * using the same interval-bucketing strategy as the Python SDK.
+   */
+  async ohlcv(options) {
+    const fromUs = toEpochUs(options.from);
+    const toUs = toEpochUs(options.to);
+    const agg = new OhlcvAggregator(options.interval);
+    for await (const event of this._readDailyEvents(
+      options.source,
+      options.market,
+      fromUs,
+      toUs,
+      (e) => e.type === "trade"
+    )) {
+      const data = event.data;
+      agg.add(event.timestamp, data.price, data.quantity);
+    }
+    return agg.finish();
+  }
+  /**
+   * Return a TradingView-shaped OHLCV response.
+   *
+   * Aggregates bars from local snapshot data and reshapes to
+   * `{ candles, volumes }`.
+   */
+  async ohlcvTradingView(options) {
+    const bars = await this.ohlcv(options);
+    return {
+      candles: bars.map((b) => ({
+        t: b.timestamp,
+        o: b.open,
+        h: b.high,
+        l: b.low,
+        c: b.close
+      })),
+      volumes: bars.map((b) => ({
+        t: b.timestamp,
+        v: b.volume,
+        trades: b.trades
+      }))
+    };
+  }
+  /**
+   * Stream historical events as an async iterable.
+   *
+   * Defaults to standardised events from local snapshots (`standard: true`).
+   * Pass `standard: false` to stream raw payloads via the `/raw` endpoint.
+   *
+   * @example
+   * ```ts
+   * for await (const row of client.replay({
+   *   source: "binance",
+   *   market: "BTC-USDT",
+   *   from: "2024-01-01T00:00:00Z",
+   *   to: "2024-01-01T01:00:00Z",
+   * })) {
+   *   console.log(row);
+   * }
+   * ```
+   */
+  async *replay(options) {
+    if (options.standard !== false) {
+      const fromUs = toEpochUs(options.from);
+      const toUs = toEpochUs(options.to);
+      yield* this._readDailyEvents(
+        options.source,
+        options.market,
+        fromUs,
+        toUs
+      );
+    } else {
+      yield* this._streamRaw(options);
+    }
+  }
+  // -----------------------------------------------------------------------
+  // Raw (API-only, not snapshot-first)
+  // -----------------------------------------------------------------------
+  /**
+   * Return raw venue-native payloads for a time range.
+   * Requires an API key. Uses the `/raw` endpoint with pagination.
+   */
+  async raw(options) {
+    const params = buildRawParams(options);
+    return this._paginateAll("/raw", params, "required");
+  }
+  // -----------------------------------------------------------------------
+  // Downloads
+  // -----------------------------------------------------------------------
+  /**
+   * Download a single snapshot file by key.
+   *
+   * Returns the native `Response` so you can consume the body as needed
+   * (`.arrayBuffer()`, `.blob()`, or pipe to a writable stream).
+   */
+  async downloadSnapshot(options) {
+    const params = { key: options.key };
+    if (options.filename) params.filename = options.filename;
+    if (options.mode) params.mode = options.mode;
+    const { response, body } = await this._fetchRaw("/snapshots/download", {
+      params,
+      auth: "if-available"
+    });
+    assertOk(response, body);
+    return response;
+  }
+  /**
+   * Get a pre-signed download URL for a snapshot file
+   * without fetching the file itself.
+   */
+  async getSnapshotDownloadUrl(options) {
+    const params = { key: options.key, mode: "url" };
+    if (options.filename) params.filename = options.filename;
+    return this._getJson("/snapshots/download", {
+      params,
+      auth: "if-available"
+    });
+  }
+  // -----------------------------------------------------------------------
+  // Lifecycle
+  // -----------------------------------------------------------------------
+  /** Release resources. Currently a no-op (reserved for future use). */
+  close() {
+  }
+  /** Async disposable support (Node ≥ 18 / TypeScript ≥ 5.2). */
+  async [Symbol.asyncDispose]() {
+    this.close();
+  }
+  // -----------------------------------------------------------------------
+  // Internals – snapshot-first local reads
+  // -----------------------------------------------------------------------
+  /**
+   * Core routine: ensure daily `.jsonl.zst` artifacts exist, decompress them,
+   * and yield matching events one at a time.
+   */
+  async *_readDailyEvents(source, market, fromUs, toUs, filter) {
+    const layout = await this._getLayout();
+    const dates = datesInRange(fromUs, toUs);
+    const filePaths = await this._ensureDailyArtifacts(
+      source,
+      market,
+      dates,
+      layout
+    );
+    for (const filePath of filePaths) {
+      const lines = await readDailyLines(filePath);
+      for (const line of lines) {
+        let event;
+        try {
+          event = JSON.parse(line);
+        } catch {
+          continue;
+        }
+        const ts = event.timestamp;
+        if (ts < fromUs || ts >= toUs) continue;
+        if (filter && !filter(event)) continue;
+        yield event;
+      }
+    }
+  }
+  /**
+   * Ensure every requested date has a materialised daily artifact.
+   * Downloads missing snapshots and materialises them into `daily/`.
+   */
+  async _ensureDailyArtifacts(source, market, dates, layout) {
+    const paths = [];
+    for (const date of dates) {
+      const dailyPath = dailyFilePath(layout.dailyDir, source, market, date);
+      if (await fileExists(dailyPath)) {
+        paths.push(dailyPath);
+        continue;
+      }
+      const snapshots = await this.listSnapshots({
+        source,
+        market,
+        from: `${date}T00:00:00Z`,
+        to: `${date}T23:59:59Z`
+      });
+      const entry = snapshots.find((s) => s.date === date);
+      if (!entry) {
+        throw new PolarisError(
+          `No snapshot available for ${source}/${market} on ${date}`
+        );
+      }
+      await this._downloadAndMaterialise(entry.key, dailyPath, layout);
+      paths.push(dailyPath);
+    }
+    return paths;
+  }
+  /**
+   * Download a snapshot to `data/` and create a hardlink (or copy) into
+   * `daily/`.
+   */
+  async _downloadAndMaterialise(key, dailyPath, layout) {
+    const dataPath = dataFilePath(layout.dataDir, key);
+    if (!await fileExists(dataPath)) {
+      const urlInfo = await this.getSnapshotDownloadUrl({ key });
+      const response = await this._fetch(urlInfo.url);
+      if (!response.ok) {
+        throw new PolarisError(
+          `Failed to download snapshot ${key}: HTTP ${response.status}`
+        );
+      }
+      const buffer = Buffer.from(await response.arrayBuffer());
+      await mkdir2(dirname2(dataPath), { recursive: true });
+      await writeFile(dataPath, buffer);
+    }
+    await linkOrCopy(dataPath, dailyPath);
+  }
+  // -----------------------------------------------------------------------
+  // Internals – raw endpoint streaming
+  // -----------------------------------------------------------------------
+  async *_streamRaw(options) {
+    const params = {
+      source: options.source,
+      market: options.market,
+      from: toIso8601(options.from),
+      to: toIso8601(options.to)
+    };
+    let cursor;
+    do {
+      if (cursor) params.cursor = cursor;
+      const res = await this._getJson("/raw", {
+        params,
+        auth: "required"
+      });
+      for (const item of res.data) {
+        yield item;
+      }
+      cursor = res.next_cursor ?? void 0;
+    } while (cursor);
+  }
+  // -----------------------------------------------------------------------
+  // Internals – HTTP layer
+  // -----------------------------------------------------------------------
+  async _getJson(path, opts = {}) {
+    const { response, body } = await this._fetchRaw(path, {
+      ...opts,
+      headers: { Accept: "application/json", ...opts.headers }
+    });
+    assertOk(response, body);
+    let json;
+    try {
+      json = JSON.parse(body);
+    } catch {
+      throw new PolarisError("Failed to parse response as JSON");
+    }
+    if (typeof json !== "object" || json === null || Array.isArray(json)) {
+      throw new PolarisError("Expected a JSON object response");
+    }
+    return json;
+  }
+  async _fetchRaw(path, opts = {}) {
+    const headers = this._buildHeaders(opts.auth ?? "none", opts.headers);
+    const url = this._buildUrl(path, opts.params);
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this._timeout);
+    let response;
+    try {
+      response = await this._fetch(url, {
+        headers,
+        signal: controller.signal,
+        redirect: "follow"
+      });
+    } catch (e) {
+      clearTimeout(timer);
+      if (e instanceof Error && e.name === "AbortError") {
+        throw new PolarisError("Request timed out");
+      }
+      throw new PolarisError(`Request failed: ${e}`);
+    }
+    clearTimeout(timer);
+    const body = await response.text();
+    return { response, body };
+  }
+  _buildHeaders(authMode, extra) {
+    const out = {
+      "User-Agent": `polaris-ts/${VERSION}`,
+      ...extra
+    };
+    if (authMode === "required" && !this._apiKey) {
+      throw new UnauthorizedError("API key is required for this endpoint");
+    }
+    if (this._apiKey && authMode !== "none") {
+      out["Authorization"] = `Bearer ${this._apiKey}`;
+    }
+    return out;
+  }
+  _buildUrl(path, params) {
+    const url = new URL(path, this._baseUrl);
+    if (params) {
+      for (const [k, v] of Object.entries(params)) {
+        if (v !== void 0) url.searchParams.set(k, v);
+      }
+    }
+    return url.toString();
+  }
+  // -----------------------------------------------------------------------
+  // Internals – pagination
+  // -----------------------------------------------------------------------
+  async _paginateAll(path, baseParams, auth) {
+    const items = [];
+    let cursor;
+    do {
+      const params = cursor ? { ...baseParams, cursor } : { ...baseParams };
+      const res = await this._getJson(path, {
+        params,
+        auth
+      });
+      items.push(...res.data);
+      cursor = res.next_cursor ?? void 0;
+    } while (cursor);
+    return items;
+  }
+  // -----------------------------------------------------------------------
+  // Internals – layout lazy init
+  // -----------------------------------------------------------------------
+  async _getLayout() {
+    if (!this._layout) this._layout = await ensureLayout(this._root);
+    return this._layout;
+  }
+};
+function readEnvApiKey() {
+  try {
+    return process?.env?.POLARIS_API_KEY;
+  } catch {
+    return void 0;
+  }
+}
+function buildRawParams(options) {
+  const p = {
+    source: options.source,
+    market: options.market
+  };
+  if (options.from !== void 0) p.from = toIso8601(options.from);
+  if (options.to !== void 0) p.to = toIso8601(options.to);
+  if (options.limit !== void 0) p.limit = String(options.limit);
+  if (options.format) p.format = options.format;
+  return p;
+}
+function buildSnapshotParams(options) {
+  const p = {
+    source: options.source,
+    market: options.market
+  };
+  if (options.from !== void 0) p.from = toIso8601(options.from);
+  if (options.to !== void 0) p.to = toIso8601(options.to);
+  if (options.limit !== void 0) p.limit = String(options.limit);
+  return p;
+}
+function assertOk(response, body) {
+  if (response.ok) return;
+  let message = `HTTP ${response.status}`;
+  let resetAt;
+  try {
+    const json = JSON.parse(body);
+    if (typeof json === "object" && json !== null) {
+      message = String(json.error ?? json.message ?? message);
+      resetAt = json.reset_at;
+    }
+  } catch {
+  }
+  switch (response.status) {
+    case 401:
+      throw new UnauthorizedError(message, response.status, body);
+    case 404:
+      throw new NotFoundError(message, response.status, body);
+    case 429:
+      throw new RateLimitedError(message, response.status, body, resetAt);
+    default:
+      throw new PolarisError(message, response.status, body);
+  }
+}
+async function readDailyLines(filePath) {
+  const compressed = await readFile(filePath);
+  const decompressed = decompress(compressed);
+  const text = new TextDecoder().decode(decompressed);
+  return text.split("\n").filter((l) => l.trim().length > 0);
+}
+export {
+  DownloadNotAllowedError,
+  NotFoundError,
+  OhlcvAggregator,
+  PolarisClient,
+  PolarisError,
+  RateLimitedError,
+  StreamDecodeError,
+  UnauthorizedError
+};
+//# sourceMappingURL=index.js.map