tradelab 0.1.0

package/src/data/csv.js

@@ -0,0 +1,340 @@
+import fs from "fs";
+import path from "path";
+
+function safeSegment(value) {
+  return String(value).replace(/[^-_.A-Za-z0-9]/g, "_");
+}
+
+function resolveDate(value, customDateParser) {
+  if (value === undefined || value === null || value === "") {
+    throw new Error("Missing date value");
+  }
+
+  if (typeof customDateParser === "function") {
+    const parsed = customDateParser(value);
+    const time = parsed instanceof Date ? parsed.getTime() : Number(parsed);
+    if (Number.isFinite(time)) return time;
+  }
+
+  if (value instanceof Date) {
+    const time = value.getTime();
+    if (Number.isFinite(time)) return time;
+  }
+
+  const raw = String(value).trim().replace(/^['"]|['"]$/g, "");
+  const numeric = Number(raw);
+  if (Number.isFinite(numeric)) {
+    return numeric < 1e11 ? numeric * 1000 : numeric;
+  }
+
+  const parsed = Date.parse(raw);
+  if (Number.isFinite(parsed)) return parsed;
+
+  const mt = raw.match(/^(\d{4})\.(\d{2})\.(\d{2})\s+(\d{2}):(\d{2})(?::(\d{2}))?$/);
+  if (mt) {
+    const [, year, month, day, hour, minute, second = "0"] = mt;
+    return new Date(
+      Number(year),
+      Number(month) - 1,
+      Number(day),
+      Number(hour),
+      Number(minute),
+      Number(second)
+    ).getTime();
+  }
+
+  throw new Error(`Cannot parse date: ${raw}`);
+}
+
+function parseCsvLine(line, delimiter) {
+  const out = [];
+  let current = "";
+  let inQuotes = false;
+
+  for (let index = 0; index < line.length; index += 1) {
+    const char = line[index];
+    if (char === '"') {
+      if (inQuotes && line[index + 1] === '"') {
+        current += '"';
+        index += 1;
+      } else {
+        inQuotes = !inQuotes;
+      }
+      continue;
+    }
+
+    if (!inQuotes && char === delimiter) {
+      out.push(current.trim());
+      current = "";
+      continue;
+    }
+
+    current += char;
+  }
+
+  out.push(current.trim());
+  return out.map((value) => value.replace(/^['"]|['"]$/g, ""));
+}
+
+function buildHeaderIndex(headers) {
+  const map = new Map();
+  headers.forEach((header, index) => {
+    map.set(header.trim().toLowerCase(), index);
+  });
+  return map;
+}
+
+function resolveColumn(column, headerIndex, aliases = []) {
+  if (typeof column === "number" && column >= 0) return column;
+
+  const candidates = [column, ...aliases]
+    .filter((value) => value !== undefined && value !== null)
+    .map((value) => String(value).trim().toLowerCase());
+
+  for (const candidate of candidates) {
+    if (headerIndex.has(candidate)) return headerIndex.get(candidate);
+  }
+
+  return -1;
+}
+
+function normalizeDateBoundary(value, fallback) {
+  if (!value) return fallback;
+  if (value instanceof Date) return value.getTime();
+  const parsed = Date.parse(String(value));
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+export function normalizeCandles(candles) {
+  if (!Array.isArray(candles)) return [];
+
+  const normalized = candles
+    .map((bar) => {
+      try {
+        const time = resolveDate(bar?.time ?? bar?.timestamp ?? bar?.date);
+        const open = Number(bar?.open ?? bar?.o);
+        const high = Number(bar?.high ?? bar?.h);
+        const low = Number(bar?.low ?? bar?.l);
+        const close = Number(bar?.close ?? bar?.c);
+        const volume = Number(bar?.volume ?? bar?.v ?? 0);
+
+        if (
+          !Number.isFinite(time) ||
+          !Number.isFinite(open) ||
+          !Number.isFinite(high) ||
+          !Number.isFinite(low) ||
+          !Number.isFinite(close)
+        ) {
+          return null;
+        }
+
+        return {
+          time,
+          open,
+          high: Math.max(high, open, close),
+          low: Math.min(low, open, close),
+          close,
+          volume: Number.isFinite(volume) ? volume : 0,
+        };
+      } catch {
+        return null;
+      }
+    })
+    .filter(Boolean)
+    .sort((left, right) => left.time - right.time);
+
+  const deduped = [];
+  let lastTime = null;
+  for (const candle of normalized) {
+    if (candle.time === lastTime) continue;
+    deduped.push(candle);
+    lastTime = candle.time;
+  }
+  return deduped;
+}
+
+export function loadCandlesFromCSV(filePath, options = {}) {
+  const {
+    delimiter = ",",
+    skipRows = 0,
+    hasHeader = true,
+    timeCol = "time",
+    openCol = "open",
+    highCol = "high",
+    lowCol = "low",
+    closeCol = "close",
+    volumeCol = "volume",
+    startDate,
+    endDate,
+    customDateParser,
+  } = options;
+
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`CSV file not found: ${filePath}`);
+  }
+
+  const content = fs.readFileSync(filePath, "utf8");
+  const lines = content.split(/\r?\n/).filter((line) => line.trim().length > 0);
+  if (lines.length <= skipRows) {
+    throw new Error(`CSV file is empty: ${filePath}`);
+  }
+
+  const headerRow = hasHeader ? parseCsvLine(lines[skipRows], delimiter) : [];
+  const headerIndex = buildHeaderIndex(headerRow);
+  const startRow = hasHeader ? skipRows + 1 : skipRows;
+
+  const timeIdx = resolveColumn(timeCol, headerIndex, [
+    "date",
+    "datetime",
+    "timestamp",
+    "ts",
+    "open time",
+    "opentime",
+  ]);
+  const openIdx = resolveColumn(openCol, headerIndex, ["o"]);
+  const highIdx = resolveColumn(highCol, headerIndex, ["h"]);
+  const lowIdx = resolveColumn(lowCol, headerIndex, ["l"]);
+  const closeIdx = resolveColumn(closeCol, headerIndex, ["c", "adj close"]);
+  const volumeIdx = resolveColumn(volumeCol, headerIndex, ["v", "vol", "quantity"]);
+
+  if (
+    timeIdx < 0 ||
+    openIdx < 0 ||
+    highIdx < 0 ||
+    lowIdx < 0 ||
+    closeIdx < 0
+  ) {
+    throw new Error(
+      `Could not resolve required CSV columns in ${path.basename(filePath)}`
+    );
+  }
+
+  const minTime = normalizeDateBoundary(startDate, -Infinity);
+  const maxTime = normalizeDateBoundary(endDate, Infinity);
+  const candles = [];
+
+  for (let row = startRow; row < lines.length; row += 1) {
+    const cols = parseCsvLine(lines[row], delimiter);
+    try {
+      const time = resolveDate(cols[timeIdx], customDateParser);
+      if (time < minTime || time > maxTime) continue;
+
+      const open = Number(cols[openIdx]);
+      const high = Number(cols[highIdx]);
+      const low = Number(cols[lowIdx]);
+      const close = Number(cols[closeIdx]);
+      const volume = volumeIdx >= 0 ? Number(cols[volumeIdx]) : 0;
+
+      if (
+        !Number.isFinite(open) ||
+        !Number.isFinite(high) ||
+        !Number.isFinite(low) ||
+        !Number.isFinite(close)
+      ) {
+        continue;
+      }
+
+      candles.push({
+        time,
+        open,
+        high: Math.max(high, open, close),
+        low: Math.min(low, open, close),
+        close,
+        volume: Number.isFinite(volume) ? volume : 0,
+      });
+    } catch {
+      continue;
+    }
+  }
+
+  return normalizeCandles(candles);
+}
+
+export function mergeCandles(...arrays) {
+  return normalizeCandles(arrays.flat());
+}
+
+export function candleStats(candles) {
+  if (!candles?.length) return null;
+
+  const normalized = normalizeCandles(candles);
+  const first = normalized[0];
+  const last = normalized[normalized.length - 1];
+  const gaps = [];
+  let minLow = Infinity;
+  let maxHigh = -Infinity;
+
+  for (const candle of normalized) {
+    if (candle.low < minLow) minLow = candle.low;
+    if (candle.high > maxHigh) maxHigh = candle.high;
+  }
+
+  for (let index = 1; index < Math.min(normalized.length, 500); index += 1) {
+    const delta = normalized[index].time - normalized[index - 1].time;
+    if (delta > 0) gaps.push(delta);
+  }
+
+  gaps.sort((left, right) => left - right);
+  const medianGapMs = gaps[Math.floor(gaps.length / 2)] || 0;
+
+  return {
+    count: normalized.length,
+    firstTime: new Date(first.time).toISOString(),
+    lastTime: new Date(last.time).toISOString(),
+    durationDays: (last.time - first.time) / (1000 * 60 * 60 * 24),
+    estimatedIntervalMin: Math.round(medianGapMs / 60000),
+    priceRange: {
+      low: minLow,
+      high: maxHigh,
+    },
+  };
+}
+
+export function saveCandlesToCache(
+  candles,
+  { symbol = "UNKNOWN", interval = "tf", period = "range", outDir = "output/data", source } = {}
+) {
+  const outputPath = path.join(
+    outDir,
+    `candles-${safeSegment(symbol)}-${safeSegment(interval)}-${safeSegment(period)}.json`
+  );
+  const normalized = normalizeCandles(candles);
+
+  fs.mkdirSync(outDir, { recursive: true });
+  fs.writeFileSync(
+    outputPath,
+    JSON.stringify(
+      {
+        symbol,
+        interval,
+        period,
+        source: source ?? null,
+        count: normalized.length,
+        asOf: new Date().toISOString(),
+        candles: normalized,
+      },
+      null,
+      2
+    ),
+    "utf8"
+  );
+
+  return outputPath;
+}
+
+export function cachedCandlesPath(symbol, interval, period, outDir = "output/data") {
+  const fileName = `candles-${safeSegment(symbol)}-${safeSegment(interval)}-${safeSegment(period)}.json`;
+  return path.join(outDir, fileName);
+}
+
+export function loadCandlesFromCache(symbol, interval, period, outDir = "output/data") {
+  const filePath = cachedCandlesPath(symbol, interval, period, outDir);
+  if (!fs.existsSync(filePath)) return null;
+
+  try {
+    const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
+    return Array.isArray(parsed?.candles) ? normalizeCandles(parsed.candles) : null;
+  } catch {
+    return null;
+  }
+}

package/src/data/index.js

@@ -0,0 +1,125 @@
+import path from "path";
+import { backtest as runBacktest } from "../engine/backtest.js";
+
+import {
+  cachedCandlesPath,
+  candleStats,
+  loadCandlesFromCache,
+  loadCandlesFromCSV,
+  mergeCandles,
+  normalizeCandles,
+  saveCandlesToCache,
+} from "./csv.js";
+import { fetchHistorical, fetchLatestCandle } from "./yahoo.js";
+
+function normalizeCacheDir(cacheDir) {
+  return cacheDir || path.join(process.cwd(), "output", "data");
+}
+
+function derivePeriodFromRange(startDate, endDate) {
+  if (!startDate || !endDate) return "custom";
+  const start = new Date(startDate).getTime();
+  const end = new Date(endDate).getTime();
+  if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start) return "custom";
+
+  const days = Math.max(1, Math.round((end - start) / (24 * 60 * 60 * 1000)));
+  return `${days}d`;
+}
+
+export async function getHistoricalCandles(options = {}) {
+  const {
+    source: requestedSource = "auto",
+    symbol,
+    interval = "1d",
+    period,
+    cache = true,
+    refresh = false,
+    cacheDir,
+    csv,
+    csvPath,
+    ...rest
+  } = options;
+
+  const effectiveCacheDir = normalizeCacheDir(cacheDir);
+  const source =
+    requestedSource === "auto"
+      ? csvPath || csv?.filePath || csv?.path
+        ? "csv"
+        : "yahoo"
+      : requestedSource;
+
+  if (source === "csv") {
+    const filePath = csvPath || csv?.filePath || csv?.path;
+    if (!filePath) {
+      throw new Error('CSV source requires "csvPath" or "csv.filePath"');
+    }
+
+    const candles = loadCandlesFromCSV(filePath, csv || rest);
+    if (cache && symbol) {
+      saveCandlesToCache(candles, {
+        symbol,
+        interval,
+        period:
+          period ??
+          derivePeriodFromRange(csv?.startDate ?? rest.startDate, csv?.endDate ?? rest.endDate),
+        outDir: effectiveCacheDir,
+        source: "csv",
+      });
+    }
+    return candles;
+  }
+
+  if (source !== "yahoo") {
+    throw new Error(`Unsupported data source "${source}"`);
+  }
+
+  if (!symbol) {
+    throw new Error('Yahoo source requires "symbol"');
+  }
+
+  const resolvedPeriod = period ?? "1y";
+
+  if (cache && !refresh) {
+    const cached = loadCandlesFromCache(symbol, interval, resolvedPeriod, effectiveCacheDir);
+    if (cached?.length) return cached;
+  }
+
+  const candles = await fetchHistorical(symbol, interval, resolvedPeriod, rest);
+  if (cache) {
+    saveCandlesToCache(candles, {
+      symbol,
+      interval,
+      period: resolvedPeriod,
+      outDir: effectiveCacheDir,
+      source: "yahoo",
+    });
+  }
+  return candles;
+}
+
+export async function backtestHistorical({
+  backtestOptions = {},
+  data,
+  ...legacy
+} = {}) {
+  const candles = await getHistoricalCandles(data || legacy);
+  return runBacktest({
+    candles,
+    symbol: data?.symbol ?? legacy.symbol,
+    interval: data?.interval ?? legacy.interval,
+    range: data?.period ?? legacy.period ?? "custom",
+    ...backtestOptions,
+  });
+}
+
+export {
+  cachedCandlesPath,
+  candleStats,
+  fetchHistorical,
+  fetchLatestCandle,
+  loadCandlesFromCache,
+  loadCandlesFromCSV,
+  mergeCandles,
+  normalizeCandles,
+  saveCandlesToCache,
+};

package/src/data/yahoo.js

@@ -0,0 +1,245 @@
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+const DAY_SEC = 24 * 60 * 60;
+const requestQueue = {
+  lastRequestAt: 0,
+  minDelayMs: 400,
+};
+
+function nowSec() {
+  return Math.floor(Date.now() / 1000);
+}
+
+function msToSec(value) {
+  return Math.floor(value / 1000);
+}
+
+function isIntraday(interval) {
+  return /(m|h)$/i.test(String(interval));
+}
+
+function normalizeInterval(interval) {
+  return String(interval || "1d").trim();
+}
+
+function parsePeriodToMs(period) {
+  if (typeof period === "number" && Number.isFinite(period)) return period;
+
+  const raw = String(period || "60d")
+    .trim()
+    .toLowerCase();
+  const normalized = raw
+    .replace(/months?$/, "mo")
+    .replace(/^(\d+)mon$/, "$1mo")
+    .replace(/^(\d+)mos$/, "$1mo");
+  const match = normalized.match(/^(\d+)(mo|m|h|d|w|y)$/i);
+
+  if (!match) {
+    throw new Error(`Invalid period "${period}". Use values like "5d", "60d", "1y".`);
+  }
+
+  const amount = Number(match[1]);
+  const unit = match[2].toLowerCase();
+  switch (unit) {
+    case "mo":
+      return Math.round(amount * 30.4375 * DAY_MS);
+    case "m":
+      return amount * 60 * 1000;
+    case "h":
+      return amount * 60 * 60 * 1000;
+    case "d":
+      return amount * DAY_MS;
+    case "w":
+      return amount * 7 * DAY_MS;
+    case "y":
+      return Math.round(amount * 365.25 * DAY_MS);
+    default:
+      throw new Error(`Unsupported period unit "${unit}"`);
+  }
+}
+
+function maxDaysForInterval(interval) {
+  const value = normalizeInterval(interval);
+  if (!isIntraday(value)) return 365 * 10;
+
+  if (/^\d+m$/i.test(value)) {
+    const minutes = Number(value.slice(0, -1));
+    if (minutes <= 2) return 7;
+    if (minutes <= 30) return 60;
+    if (minutes <= 60) return 730;
+    return 365;
+  }
+
+  if (/^\d+h$/i.test(value)) return 730;
+  return 60;
+}
+
+function sanitizeBars(candles) {
+  const deduped = new Map();
+  for (const candle of candles) {
+    if (
+      !Number.isFinite(candle?.time) ||
+      !Number.isFinite(candle?.open) ||
+      !Number.isFinite(candle?.high) ||
+      !Number.isFinite(candle?.low) ||
+      !Number.isFinite(candle?.close)
+    ) {
+      continue;
+    }
+
+    deduped.set(candle.time, {
+      time: candle.time,
+      open: candle.open,
+      high: Math.max(candle.high, candle.open, candle.close),
+      low: Math.min(candle.low, candle.open, candle.close),
+      close: candle.close,
+      volume: Number.isFinite(candle.volume) ? candle.volume : 0,
+    });
+  }
+
+  return [...deduped.values()].sort((left, right) => left.time - right.time);
+}
+
+async function rateLimitedFetch(url, options = {}) {
+  const elapsed = Date.now() - requestQueue.lastRequestAt;
+  if (elapsed < requestQueue.minDelayMs) {
+    await sleep(requestQueue.minDelayMs - elapsed);
+  }
+
+  requestQueue.lastRequestAt = Date.now();
+  return fetch(url, {
+    ...options,
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+      ...options.headers,
+    },
+  });
+}
+
+async function fetchYahooChart(symbol, { period1, period2, interval, includePrePost = false }) {
+  const url = new URL(
+    `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}`
+  );
+  url.searchParams.set("period1", String(Math.floor(period1)));
+  url.searchParams.set("period2", String(Math.floor(period2)));
+  url.searchParams.set("interval", normalizeInterval(interval));
+  url.searchParams.set("includePrePost", includePrePost ? "true" : "false");
+  url.searchParams.set("events", "div,splits");
+
+  const response = await rateLimitedFetch(url.toString());
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Yahoo API error ${response.status}: ${text}`);
+  }
+
+  const payload = await response.json();
+  if (payload.chart?.error) {
+    throw new Error(payload.chart.error.description || "Yahoo chart error");
+  }
+
+  const result = payload.chart?.result?.[0];
+  if (!result) return [];
+
+  const timestamps = result.timestamp || [];
+  const quote = result.indicators?.quote?.[0] || {};
+  const open = quote.open || [];
+  const high = quote.high || [];
+  const low = quote.low || [];
+  const close = quote.close || [];
+  const volume = quote.volume || [];
+
+  const candles = [];
+  for (let index = 0; index < timestamps.length; index += 1) {
+    if (
+      open[index] == null ||
+      high[index] == null ||
+      low[index] == null ||
+      close[index] == null
+    ) {
+      continue;
+    }
+
+    candles.push({
+      time: timestamps[index] * 1000,
+      open: open[index],
+      high: high[index],
+      low: low[index],
+      close: close[index],
+      volume: volume[index] ?? 0,
+    });
+  }
+
+  return candles;
+}
+
+async function fetchYahooChartWithRetry(symbol, params, maxRetries = 5) {
+  let lastError = null;
+
+  for (let attempt = 0; attempt < maxRetries; attempt += 1) {
+    try {
+      return await fetchYahooChart(symbol, params);
+    } catch (error) {
+      lastError = error;
+      const message = String(error?.message || error);
+      const isRateLimited = /too many requests|rate limit|429/i.test(message);
+      const isRetryable = isRateLimited || /timeout|fetch failed|network/i.test(message);
+
+      if (!isRetryable || attempt === maxRetries - 1) break;
+
+      const delay = isRateLimited
+        ? Math.min(30_000, 2_000 * 2 ** attempt)
+        : Math.min(10_000, 750 * (attempt + 1));
+      await sleep(delay);
+    }
+  }
+
+  throw lastError ?? new Error(`Failed to fetch Yahoo data for ${symbol}`);
+}
+
+export async function fetchHistorical(symbol, interval = "5m", period = "60d", options = {}) {
+  const normalizedInterval = normalizeInterval(interval);
+  const spanMs = parsePeriodToMs(period);
+  const maxSpanMs = maxDaysForInterval(normalizedInterval) * DAY_MS;
+  const includePrePost = Boolean(options.includePrePost);
+
+  if (spanMs <= maxSpanMs) {
+    const endSec = nowSec();
+    const startSec = Math.max(0, endSec - msToSec(spanMs));
+    const candles = await fetchYahooChartWithRetry(symbol, {
+      period1: startSec,
+      period2: endSec,
+      interval: normalizedInterval,
+      includePrePost,
+    });
+    return sanitizeBars(candles);
+  }
+
+  const chunks = [];
+  let remainingMs = spanMs;
+  let chunkEndMs = Date.now();
+
+  while (remainingMs > 0) {
+    const takeMs = Math.min(remainingMs, maxSpanMs);
+    const chunkStartMs = chunkEndMs - takeMs;
+    const candles = await fetchYahooChartWithRetry(symbol, {
+      period1: msToSec(chunkStartMs),
+      period2: msToSec(chunkEndMs),
+      interval: normalizedInterval,
+      includePrePost,
+    });
+    chunks.push(...candles);
+    chunkEndMs = chunkStartMs - 1000;
+    remainingMs -= takeMs;
+
+    if (chunks.length > 2_000_000) break;
+  }
+
+  return sanitizeBars(chunks);
+}
+
+export async function fetchLatestCandle(symbol, interval = "1m", options = {}) {
+  const bars = await fetchHistorical(symbol, interval, "5d", options);
+  return bars[bars.length - 1] ?? null;
+}