ultrastiching-cli 0.0.0-stage → 0.2.0

package/src/http.js

@@ -0,0 +1,129 @@
+import { endpoint } from './registry.js';
+import { normalizeOrigin, resolveBaseUrl, resolveKey } from './config.js';
+
+export function redactKey(value, key) {
+  if (typeof value !== 'string' || !key) return value;
+  return value.replaceAll(key, '[REDACTED_API_KEY]');
+}
+
+function filteredHeaders(headers, key) {
+  const privateNames = new Set(['authorization', 'proxy-authorization', 'set-cookie', 'cookie', 'x-api-key']);
+  return Object.fromEntries(Object.entries(headers ?? {}).map(([name, value]) => [name,
+    privateNames.has(name.toLowerCase()) ? '[REDACTED]' : redactKey(value, key)]));
+}
+
+export class MeridianHttpError extends Error {
+  constructor({ status, statusText, errorCode, requestId, body, headers, url, key }) {
+    super(`UltraStiching API returned HTTP ${status}${errorCode ? ` (${errorCode})` : ''}`);
+    this.name = 'MeridianHttpError';
+    this.status = status;
+    this.statusText = statusText;
+    this.errorCode = errorCode;
+    this.requestId = requestId;
+    this.body = body; // Original body is preserved for programmatic callers.
+    this.headers = headers;
+    this.url = url;
+    this.key = key;
+  }
+  toDiagnostic() {
+    return {
+      type: this.name, status: this.status, status_text: redactKey(this.statusText, this.key),
+      error: redactKey(this.errorCode, this.key), request_id: redactKey(this.requestId, this.key),
+      raw_body: redactKey(this.body, this.key),
+      response: { status_code: this.status, reason: redactKey(this.statusText, this.key),
+        url: redactKey(this.url, this.key), headers: filteredHeaders(this.headers, this.key),
+        body: redactKey(this.body, this.key) },
+    };
+  }
+}
+
+export class MeridianTransportError extends Error {
+  constructor(cause, key) {
+    super('UltraStiching request could not be completed', { cause });
+    this.name = 'MeridianTransportError';
+    this.key = key;
+  }
+  toDiagnostic() {
+    return { type: this.name, message: this.message, cause: {
+      type: this.cause?.name ?? 'Error', message: redactKey(String(this.cause), this.key),
+      code: this.cause?.code, errno: this.cause?.errno,
+    } };
+  }
+}
+
+export class MeridianResponseError extends Error {
+  constructor({ status, requestId, body, key, cause }) {
+    super('UltraStiching API returned a successful response with invalid JSON', { cause });
+    this.name = 'MeridianResponseError';
+    this.status = status;
+    this.requestId = requestId;
+    this.body = body;
+    this.key = key;
+  }
+  toDiagnostic() {
+    return { type: this.name, status: this.status, request_id: redactKey(this.requestId, this.key), raw_body: redactKey(this.body, this.key) };
+  }
+}
+
+export async function requestEndpoint(meta, payload, { key, baseUrl, timeout = 60000, fetchImpl = fetch } = {}) {
+  if (!meta || meta.protocol !== 'http' || !meta.path?.startsWith('/') || meta.path.startsWith('//')) {
+    throw new Error('A known HTTP registry endpoint is required');
+  }
+  if (meta.response?.content_type === 'text/event-stream') {
+    throw new Error(`${meta.name} is SSE; synchronous call is unsupported. Read its contract with ultrastiching info ${meta.name}`);
+  }
+  const apiKey = resolveKey(key);
+  const origin = baseUrl === undefined ? resolveBaseUrl() : normalizeOrigin(baseUrl);
+  const ms = Number(timeout);
+  if (!Number.isFinite(ms) || ms <= 0) throw new Error('timeout must be a positive number of milliseconds');
+  let path = meta.path;
+  const query = new URLSearchParams();
+  if (meta.method === 'GET') {
+    for (const parameter of meta.request.parameters) {
+      const value = payload[parameter.name];
+      if (value === undefined) continue;
+      if (parameter.in === 'path') {
+        if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${parameter.name} must be a positive integer`);
+        path = path.replace(`{${parameter.name}}`, encodeURIComponent(String(value)));
+      } else {
+        if (value === null) throw new Error(`${parameter.name} cannot be null in a GET query`);
+        query.set(parameter.name, String(value));
+      }
+    }
+  }
+  const url = `${origin}${path}${query.size ? `?${query}` : ''}`;
+  let response;
+  try {
+    response = await fetchImpl(url, {
+      method: meta.method,
+      headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': meta.request?.content_type ?? 'application/json' },
+      ...(meta.method === 'GET' ? {} : { body: JSON.stringify(payload) }),
+      redirect: 'manual',
+      signal: AbortSignal.timeout(ms),
+    });
+  } catch (cause) { throw new MeridianTransportError(cause, apiKey); }
+  let buffer;
+  try { buffer = Buffer.from(await response.arrayBuffer()); }
+  catch (cause) { throw new MeridianTransportError(cause, apiKey); }
+  const requestId = response.headers.get('x-meridian-request-id');
+  if (!response.ok) {
+    const body = buffer.toString('utf8');
+    let errorCode;
+    try {
+      const parsed = JSON.parse(body);
+      errorCode = typeof parsed.error === 'string' ? parsed.error : parsed.error?.code;
+    } catch { /* raw body is retained */ }
+    throw new MeridianHttpError({ status: response.status, statusText: response.statusText, errorCode, requestId, body,
+      headers: Object.fromEntries(response.headers), url: response.url || url, key: apiKey });
+  }
+  const contentType = response.headers.get('content-type')?.split(';', 1)[0].trim().toLowerCase();
+  if (contentType === 'application/vnd.apache.arrow.stream') return { binary: true, data: buffer, requestId };
+  try { return { binary: false, data: JSON.parse(buffer.toString('utf8')), requestId }; }
+  catch (cause) { throw new MeridianResponseError({ status: response.status, requestId, body: buffer.toString('utf8'), key: apiKey, cause }); }
+}
+
+export async function callByName(name, payload, options) {
+  const meta = endpoint(name, 'http');
+  if (!meta) throw new Error(`Unknown HTTP endpoint ${name}; run ultrastiching list --search ${name}`);
+  return requestEndpoint(meta, payload, options);
+}

package/src/main.js

@@ -0,0 +1,201 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { parseArgs, jsonObject, requireValue } from './args.js';
+import { resolveBaseUrl, resolveKey } from './config.js';
+import { callByName, redactKey } from './http.js';
+import { buildPayload, endpoint, loadRegistry, parseParameter, searchEndpoints } from './registry.js';
+import { streamNewsflash } from './websocket.js';
+import { credentialStatus, configureCredential, syncCredentialEnvironment, mcpHeaders } from './auth.js';
+import { readFileSync } from 'node:fs';
+
+const VERSION = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version;
+
+const HELP = `ultrastiching — UltraStiching data API CLI
+
+  ultrastiching --version
+  ultrastiching auth status
+  ultrastiching auth sync [--from-file]
+  ultrastiching auth configure [--replace]
+  ultrastiching mcp headers  (MCP client only; stdout contains credentials)
+  ultrastiching doctor [--probe]
+  ultrastiching list [--search text] [--binary] [--verbose]
+  ultrastiching info <endpoint>
+  ultrastiching call <endpoint> [--parameter value ...] [--args JSON] [--out file]
+  ultrastiching arrow <get_* handler> --args '{"params":{...},"fields":[...]}' [--out file]
+  ultrastiching ws newsflash [--limit n] [--keyword text] [--count n]
+  ultrastiching datasets
+  ultrastiching db init|sync|status|validate|repair|sql|export [options]
+
+Global: --key, --base-url, --timeout (milliseconds), --compact
+`;
+
+function emit(write, value, compact, key) {
+  write(`${redactKey(JSON.stringify(value, null, compact ? 0 : 2), key)}\n`);
+}
+
+function assertFlags(flags, allowed) {
+  const extra = Object.keys(flags).filter((name) => !allowed.has(name));
+  if (extra.length) throw new Error(`Unknown option(s): ${extra.map((name) => `--${name.replaceAll('_', '-')}`).join(', ')}`);
+}
+
+function valuesFor(meta, flags) {
+  const values = jsonObject(flags.args);
+  const names = new Set(meta.parameters);
+  const allowed = new Set([...names, ...Object.keys(meta.payload), 'args', 'out', 'key', 'base_url', 'timeout', 'compact']);
+  assertFlags(flags, allowed);
+  for (const name of names) {
+    const wireName = Object.entries(meta.payload).find(([, value]) => value?.$param === name)?.[0] ?? name;
+    const flagName = Object.hasOwn(flags, name) ? name : wireName;
+    if (!Object.hasOwn(flags, flagName)) continue;
+    const field = meta.request?.fields?.find((item) => item.name === wireName) ?? { name };
+    values[name] = parseParameter(requireValue(flags, flagName), field);
+  }
+  return values;
+}
+
+function outputResult(name, result, flags, write, key) {
+  if (result.binary) {
+    const target = resolve(flags.out ?? `meridian-out/${name}-${Date.now()}.arrow`);
+    mkdirSync(dirname(target), { recursive: true });
+    writeFileSync(target, result.data);
+    emit(write, { path: target, bytes: result.data.length, format: 'arrow_ipc', request_id: result.requestId }, flags.compact, key);
+  } else if (flags.out || rowCount(result.data) > 500) {
+    const target = resolve(flags.out ? requireValue(flags, 'out') : `meridian-out/${name}-${Date.now()}.json`);
+    mkdirSync(dirname(target), { recursive: true });
+    writeFileSync(target, `${JSON.stringify(result.data, null, flags.compact ? 0 : 2)}\n`);
+    emit(write, { path: target, format: 'json', request_id: result.requestId }, flags.compact, key);
+  } else emit(write, result.data, flags.compact, key);
+}
+
+function rowCount(data) {
+  if (Array.isArray(data)) return data.length;
+  if (!data || typeof data !== 'object') return 0;
+  if (Array.isArray(data.data_frame)) return data.data_frame.length;
+  if (Array.isArray(data.data)) return data.data.length;
+  if (Array.isArray(data.items)) return data.items.length;
+  return 0;
+}
+
+export async function main(argv, { write = (line) => process.stdout.write(line), fetchImpl, socketFactory } = {}) {
+  const { flags, positional } = parseArgs(argv);
+  const [command, name] = positional;
+  if (flags.version && !command) { assertFlags(flags, new Set(['version'])); write(`${VERSION}\n`); return; }
+  if (!command || flags.help) { write(HELP); return; }
+  if (command === 'auth') {
+    if (positional.length !== 2) throw new Error('Use ultrastiching auth status|sync|configure');
+    assertFlags(flags, new Set(name === 'configure' ? ['replace', 'compact'] : name === 'sync' ? ['from_file', 'compact'] : ['compact']));
+    if (name === 'configure') {
+      if (!process.stdin.isTTY || !process.stdout.isTTY) throw new Error('请在交互式终端运行 ultrastiching auth configure；Key 不能从参数或管道传入。');
+      emit(write, await configureCredential({ replace: !!flags.replace }), flags.compact);
+    } else if (name === 'status') emit(write, credentialStatus(), flags.compact);
+    else if (name === 'sync') emit(write, syncCredentialEnvironment({ fromFile: !!flags.from_file }), flags.compact);
+    else throw new Error('Use ultrastiching auth status|sync|configure');
+    return;
+  }
+  if (command === 'mcp') {
+    assertFlags(flags, new Set());
+    if (name !== 'headers' || positional.length !== 2) throw new Error('Use ultrastiching mcp headers (MCP client only)');
+    write(`${JSON.stringify(mcpHeaders())}\n`);
+    return;
+  }
+  if (command === 'list') {
+    assertFlags(flags, new Set(['search', 'binary', 'verbose', 'compact']));
+    const items = searchEndpoints({ search: flags.search === undefined ? '' : requireValue(flags, 'search'), binary: !!flags.binary });
+    emit(write, { total: items.length, endpoints: items.map((item) => flags.verbose ? {
+      name: item.name, title: item.title, protocol: item.protocol, path: item.path,
+      response_format: item.response?.return_format,
+    } : { name: item.name, title: item.title }) }, flags.compact);
+    return;
+  }
+  if (command === 'info') {
+    assertFlags(flags, new Set(['compact']));
+    const meta = endpoint(name);
+    if (!meta) throw new Error(`Unknown endpoint ${name}; run ultrastiching list --search ${name}`);
+    emit(write, meta, flags.compact);
+    return;
+  }
+  if (command === 'doctor') {
+    assertFlags(flags, new Set(['key', 'base_url', 'timeout', 'probe', 'compact']));
+    const registry = loadRegistry();
+    const key = resolveKey(flags.key, { optional: true });
+    const result = {
+      node: process.version, base_url: resolveBaseUrl(flags.base_url),
+      api_key_configured: !!key, registry: registry.protocol_counts,
+      duckdb: { available: false }, probe: 'not_requested',
+    };
+    try {
+      const duckdb = await import('@duckdb/node-api');
+      result.duckdb = { available: true, version: duckdb.default.version() };
+    } catch (error) {
+      result.duckdb = { available: false, error: error.message };
+    }
+    if (flags.probe) {
+      if (!key) throw new Error('doctor --probe requires an API Key');
+      const meta = endpoint('get_s_stockname', 'http');
+      const payload = buildPayload(meta, { code: ['600745.SH'] });
+      const response = await callByName(meta.name, payload, { key, baseUrl: flags.base_url, timeout: flags.timeout, fetchImpl });
+      result.probe = { status: 'ok', request_id: response.requestId };
+    }
+    emit(write, result, flags.compact, key);
+    return;
+  }
+  if (command === 'call') {
+    const meta = endpoint(name, 'http');
+    if (!meta) throw new Error(`Unknown HTTP endpoint ${name}; run ultrastiching list --search ${name}`);
+    const values = valuesFor(meta, flags);
+    const key = resolveKey(flags.key);
+    const response = await callByName(name, buildPayload(meta, values), {
+      key, baseUrl: flags.base_url, timeout: flags.timeout, fetchImpl,
+    });
+    outputResult(name, response, flags, write, key);
+    return;
+  }
+  if (command === 'arrow') {
+    assertFlags(flags, new Set(['args', 'out', 'key', 'base_url', 'timeout', 'compact']));
+    const handler = endpoint(name, 'http');
+    if (!handler || !name.startsWith('get_')) throw new Error('arrow requires a registered get_* handler; use call for native Arrow endpoints');
+    const args = jsonObject(flags.args);
+    const meta = endpoint('arrow', 'http');
+    const payload = buildPayload(meta, { handler: name, ...args });
+    const key = resolveKey(flags.key);
+    const response = await callByName('arrow', payload, {
+      key, baseUrl: flags.base_url, timeout: flags.timeout, fetchImpl,
+    });
+    if (!response.binary) throw new Error(`arrow returned ${typeof response.data}, expected Arrow IPC stream`);
+    outputResult(name, response, flags, write, key);
+    return;
+  }
+  if (command === 'ws') {
+    if (name !== 'newsflash') throw new Error('ws currently supports newsflash');
+    assertFlags(flags, new Set(['cursor', 'limit', 'keyword', 'min_importance', 'count', 'idle_timeout', 'key', 'ws_base_url', 'compact']));
+    const meta = endpoint('ws_newsflash', 'websocket');
+    const values = {};
+    for (const param of meta.parameters) {
+      if (!Object.hasOwn(flags, param)) continue;
+      const field = meta.request?.fields?.find((item) => item.name === param) ?? { name: param };
+      values[param] = parseParameter(requireValue(flags, param), field);
+    }
+    const key = resolveKey(flags.key);
+    const result = await streamNewsflash(values, {
+      key, websocketBaseUrl: flags.ws_base_url, count: flags.count,
+      idleTimeout: flags.idle_timeout, socketFactory, onMessage: (line) => write(`${redactKey(line, key)}\n`),
+    });
+    emit(write, result, flags.compact, key);
+    return;
+  }
+  if (command === 'datasets') {
+    assertFlags(flags, new Set(['compact']));
+    const { datasets } = await import('./marketdb.js');
+    emit(write, { datasets: datasets() }, flags.compact);
+    return;
+  }
+  if (command === 'db') {
+    const { dbCommand } = await import('./db-cli.js');
+    const result = await dbCommand(name, flags, { fetchImpl });
+    const key = flags.key === undefined ? undefined : resolveKey(flags.key);
+    emit(write, result, flags.compact, key);
+    return result.status === 'incomplete' || result.ok === false || result.repaired?.some((item) => item.status === 'incomplete')
+      ? { exitCode: 2 } : undefined;
+  }
+  throw new Error(`Unknown command ${command}; run ultrastiching --help`);
+}