routstrd 0.3.1 → 0.3.3

package/src/daemon/models.ts

@@ -120,9 +120,64 @@ export function createModelService(modelManager: ModelManager, store: SdkStore)
     };
   };
 
+  /**
+   * Force-refresh everything: re-fetch Nostr provider discovery events,
+   * routstr21 model list, Nostr review events, and models from all enabled
+   * providers. Syncs the discovered provider list into the store.
+   */
+  const refreshProvidersAndModels = async (): Promise<void> => {
+    // Reset the bootstrap promise so we don't reuse cached results
+    providerBootstrapPromise = null;
+
+    console.log("Force-refreshing providers from Nostr...");
+
+    // Force-refresh provider discovery from Nostr (kind 38421)
+    const providers = await modelManager.bootstrapProviders(false, true);
+    console.log(`Discovered ${providers.length} providers from Nostr`);
+
+    // Force-refresh routstr21 models from Nostr (kind 38423)
+    const routstr21ModelIds = await modelManager.fetchRoutstr21Models(true);
+    console.log(`Fetched ${routstr21ModelIds.length} routstr21 model IDs from Nostr`);
+
+    // Force-refresh models from all providers
+    const models = await modelManager.fetchModels(providers, true);
+    console.log(`Fetched ${models.length} models from ${providers.length} providers`);
+
+    // Sync review events from Nostr (kind 38425) and apply disabled status
+    const reviewedDisabled = await modelManager.syncReviewedProvidersFromNostr(
+      providers,
+      undefined,
+      true,
+    );
+    if (reviewedDisabled.length > 0) {
+      console.log(
+        `Review sync disabled ${reviewedDisabled.length} provider(s): ${reviewedDisabled.join(", ")}`,
+      );
+    }
+
+    // Sync discovered providers into the store
+    const { baseUrlsList, setBaseUrlsList, disabledProviders, setDisabledProviders } =
+      store.getState() as any;
+
+    // Replace baseUrlsList with the fresh provider list
+    setBaseUrlsList(providers);
+
+    // Merge review-disabled providers into the store's disabled list
+    const existingDisabled = new Set(disabledProviders || []);
+    for (const url of reviewedDisabled) {
+      existingDisabled.add(url);
+    }
+    setDisabledProviders([...existingDisabled]);
+
+    console.log(
+      `Provider refresh complete: ${providers.length} total, ${existingDisabled.size} disabled`,
+    );
+  };
+
   return {
     ensureProvidersBootstrapped,
     getRoutstr21Models,
     getModelProviders,
+    refreshProvidersAndModels,
   };
 }

package/src/daemon/request-response-log-sink.ts

@@ -0,0 +1,269 @@
+import { createWriteStream, mkdirSync, type WriteStream } from "fs";
+import { writeFile } from "fs/promises";
+import { join } from "path";
+import type { SdkLogger } from "@routstr/sdk";
+
+export interface RequestResponseLogRequestInput {
+  method: string;
+  url: string;
+  path: string;
+  baseUrl: string;
+  headers: Record<string, string>;
+  body?: unknown;
+  rawBody?: string;
+}
+
+export interface RequestResponseLogSink {
+  logRequest?(input: RequestResponseLogRequestInput): string | undefined | Promise<string | undefined>;
+  logResponseStart?(id: string | undefined, response: Response): void | Promise<void>;
+  logResponseChunk?(id: string | undefined, sequence: number, text: string): void | Promise<void>;
+  logResponseEnd?(id: string | undefined): void | Promise<void>;
+  logResponseError?(id: string | undefined, error: unknown): void | Promise<void>;
+  logResponseBody?(id: string | undefined, response: Response): void | Promise<void>;
+}
+
+interface ActiveResponseLog {
+  stream: WriteStream;
+  pending: Promise<void>;
+}
+
+export interface FileRequestResponseLogSinkOptions {
+  dir: string;
+  logger?: SdkLogger;
+}
+
+const SENSITIVE_HEADER_NAMES = new Set([
+  "authorization",
+  "x-cashu",
+  "cookie",
+  "set-cookie",
+  "proxy-authorization",
+]);
+
+const SENSITIVE_BODY_FIELD_NAMES = new Set([
+  "authorization",
+  "api_key",
+  "apikey",
+  "apiKey",
+  "access_token",
+  "accessToken",
+  "bearer",
+  "cashu",
+  "cookie",
+  "key",
+  "password",
+  "secret",
+  "token",
+  "x-cashu",
+]);
+
+const REDACTED = "[REDACTED]";
+
+const sanitizeForFilename = (value: string): string =>
+  value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
+
+const makeId = (): string => {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const random = crypto.randomUUID().slice(0, 8);
+  return `${timestamp}-${random}`;
+};
+
+const headersToObject = (headers: Headers): Record<string, string> => {
+  const out: Record<string, string> = {};
+  headers.forEach((value, key) => {
+    out[key] = value;
+  });
+  return out;
+};
+
+const redactHeaders = (headers: Record<string, string>): Record<string, string> =>
+  Object.fromEntries(
+    Object.entries(headers).map(([key, value]) => [
+      key,
+      SENSITIVE_HEADER_NAMES.has(key.toLowerCase()) ? REDACTED : value,
+    ]),
+  );
+
+const redactBody = (value: unknown): unknown => {
+  if (!value || typeof value !== "object") return value;
+  if (Array.isArray(value)) return value.map(redactBody);
+
+  return Object.fromEntries(
+    Object.entries(value as Record<string, unknown>).map(([key, entry]) => [
+      key,
+      SENSITIVE_BODY_FIELD_NAMES.has(key) || SENSITIVE_BODY_FIELD_NAMES.has(key.toLowerCase())
+        ? REDACTED
+        : redactBody(entry),
+    ]),
+  );
+};
+
+const redactRawBody = (rawBody: string | undefined): string | undefined => {
+  if (!rawBody) return undefined;
+  try {
+    return JSON.stringify(redactBody(JSON.parse(rawBody)));
+  } catch {
+    return rawBody;
+  }
+};
+
+export class FileRequestResponseLogSink implements RequestResponseLogSink {
+  private requestsDir: string;
+  private responsesDir: string;
+  private activeResponses = new Map<string, ActiveResponseLog>();
+
+  constructor(private options: FileRequestResponseLogSinkOptions) {
+    this.requestsDir = join(options.dir, "requests");
+    this.responsesDir = join(options.dir, "responses");
+    mkdirSync(this.requestsDir, { recursive: true });
+    mkdirSync(this.responsesDir, { recursive: true });
+  }
+
+  async logRequest(input: RequestResponseLogRequestInput): Promise<string | undefined> {
+    try {
+      const id = makeId();
+      const filePath = join(this.requestsDir, `${sanitizeForFilename(id)}.json`);
+      await writeFile(
+        filePath,
+        JSON.stringify(
+          {
+            id,
+            timestamp: new Date().toISOString(),
+            method: input.method,
+            url: input.url,
+            path: input.path,
+            baseUrl: input.baseUrl,
+            headers: redactHeaders(input.headers),
+            body: redactBody(input.body),
+            rawBody: redactRawBody(input.rawBody),
+          },
+          null,
+          2,
+        ),
+      );
+      return id;
+    } catch (error) {
+      this.options.logger?.error?.("[request-response-log] failed to log request:", error);
+      return undefined;
+    }
+  }
+
+  async logResponseStart(id: string | undefined, response: Response): Promise<void> {
+    if (!id) return;
+    await this.append(id, {
+      type: "response_start",
+      status: response.status,
+      statusText: response.statusText,
+      headers: redactHeaders(headersToObject(response.headers)),
+    });
+  }
+
+  logResponseChunk(id: string | undefined, sequence: number, text: string): void {
+    if (!id) return;
+    void this.append(id, {
+      type: "chunk",
+      sequence,
+      text,
+    });
+  }
+
+  async logResponseEnd(id: string | undefined): Promise<void> {
+    if (!id) return;
+    await this.append(id, { type: "end" });
+    await this.close(id);
+  }
+
+  async logResponseError(id: string | undefined, error: unknown): Promise<void> {
+    if (!id) return;
+    await this.append(id, {
+      type: "error",
+      error: error instanceof Error ? { message: error.message, stack: error.stack } : String(error),
+    });
+    await this.close(id);
+  }
+
+  async logResponseBody(id: string | undefined, response: Response): Promise<void> {
+    if (!id) return;
+    try {
+      if (!response.body) {
+        await this.logResponseEnd(id);
+        return;
+      }
+
+      const reader = response.body.getReader();
+      const decoder = new TextDecoder("utf-8");
+      let sequence = 0;
+
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        if (value && value.byteLength > 0) {
+          await this.append(id, {
+            type: "chunk",
+            sequence: sequence++,
+            text: decoder.decode(value, { stream: true }),
+          });
+        }
+      }
+
+      const tail = decoder.decode();
+      if (tail) {
+        await this.append(id, {
+          type: "chunk",
+          sequence: sequence++,
+          text: tail,
+        });
+      }
+
+      await this.logResponseEnd(id);
+    } catch (error) {
+      await this.logResponseError(id, error);
+    }
+  }
+
+  private getOrCreate(id: string): ActiveResponseLog {
+    const existing = this.activeResponses.get(id);
+    if (existing) return existing;
+
+    const filePath = join(this.responsesDir, `${sanitizeForFilename(id)}.jsonl`);
+    const stream = createWriteStream(filePath, { flags: "a" });
+    const active: ActiveResponseLog = {
+      stream,
+      pending: Promise.resolve(),
+    };
+
+    stream.on("error", (error) => {
+      this.options.logger?.error?.("[request-response-log] response log stream error:", error);
+    });
+
+    this.activeResponses.set(id, active);
+    return active;
+  }
+
+  private async append(id: string, event: Record<string, unknown>): Promise<void> {
+    try {
+      const active = this.getOrCreate(id);
+      const line = JSON.stringify({ requestLogId: id, timestamp: new Date().toISOString(), ...event }) + "\n";
+      active.pending = active.pending.then(
+        () =>
+          new Promise<void>((resolve, reject) => {
+            active.stream.write(line, (error) => {
+              if (error) reject(error);
+              else resolve();
+            });
+          }),
+      );
+      await active.pending;
+    } catch (error) {
+      this.options.logger?.error?.("[request-response-log] failed to append response event:", error);
+    }
+  }
+
+  private async close(id: string): Promise<void> {
+    const active = this.activeResponses.get(id);
+    if (!active) return;
+    this.activeResponses.delete(id);
+    await active.pending;
+    await new Promise<void>((resolve) => active.stream.end(resolve));
+  }
+}

package/src/daemon/types.ts

@@ -11,6 +11,12 @@ export type UsageData = {
   totalTokens: number;
   cost: number;
   satsCost: number;
+  provider?: string;
+  inputMsats?: number;
+  outputMsats?: number;
+  totalMsats?: number;
+  cacheReadInputTokens?: number;
+  cacheCreationInputTokens?: number;
 };
 
 export type UsageTrackingEntry = UsageData & {

package/src/tui/usage/app.ts

@@ -1,6 +1,6 @@
 import { getVisibleTabs } from "./constants.ts";
 import type { Tab } from "./types.ts";
-import { fetchBalance, fetchClients, fetchStatus, fetchUsage, hasAnyNpubs, isDaemonRunning, type BalanceInfo, type ClientInfo, type StatusInfo } from "./data.ts";
+import { fetchBalance, fetchClients, fetchStatus, fetchUsageSummary, hasAnyNpubs, isDaemonRunning, type BalanceInfo, type ClientInfo, type StatusInfo } from "./data.ts";
 import {
   applyScrollToContent,
   exitSearchMode,
@@ -89,7 +89,7 @@ export async function runUsageTui(): Promise<void> {
       const height = getHeight();
 
       if (forceFetch || shouldUpdate) {
-        stats = await fetchUsage(10000);
+        stats = await fetchUsageSummary();
         balance = await fetchBalance();
         status = await fetchStatus();
         clients = await fetchClients();

package/src/tui/usage/data.ts

@@ -1,6 +1,6 @@
 import type { UsageTrackingEntry } from "../../daemon/types.ts";
 import { callDaemon, isDaemonRunning } from "../../utils/daemon-client.ts";
-import type { ClientStats, DayStats, ModelStats, NpubStats, ProviderStats, UsageStats } from "./types.ts";
+import type { UsageStats, UsageSummary } from "./types.ts";
 
 export { isDaemonRunning };
 
@@ -76,53 +76,35 @@ export async function fetchBalance(): Promise<BalanceInfo | null> {
   }
 }
 
-export async function fetchUsage(limit = 10000): Promise<UsageStats | null> {
+export async function fetchUsageSummary(): Promise<UsageStats | null> {
   try {
     const running = await isDaemonRunning();
     if (!running) return null;
 
-    const result = await callDaemon(`/usage?limit=${limit}`);
+    const tz = new Date().getTimezoneOffset();
+    const result = await callDaemon(`/usage/summary?tz=${tz}`);
     if (result.error) return null;
 
-    // The auth proxy filters usage to the authenticated npub's clients and
-    // returns client IDs without the owner suffix.
-    const entries = result.output as UsageTrackingEntry[] | undefined;
-    const visibleEntries = Array.isArray(entries) ? entries : [];
-
-    // Calculate totals from visible entries
-    const totals = visibleEntries.reduce(
-      (acc, entry) => ({
-        promptTokens: acc.promptTokens + entry.promptTokens,
-        completionTokens: acc.completionTokens + entry.completionTokens,
-        totalTokens: acc.totalTokens + entry.totalTokens,
-        satsCost: acc.satsCost + entry.satsCost,
-      }),
-      { promptTokens: 0, completionTokens: 0, totalTokens: 0, satsCost: 0 },
-    );
+    const summary = result.output as UsageSummary | undefined;
+    if (!summary || typeof summary.totals !== "object") return null;
 
     return {
-      entries: visibleEntries,
-      totalEntries: visibleEntries.length,
-      totalSatsCost: totals.satsCost,
-      recentSatsCost: totals.satsCost, // For now, recent = total since we don't have time window
-      limit,
+      entries: summary.recent,
+      totalEntries: summary.totals.requests,
+      totalSatsCost: summary.totals.satsCost,
+      recentSatsCost: summary.totals.satsCost,
+      limit: 50,
+      summary,
     };
   } catch {
     return null;
   }
 }
 
-export function getTodayStart(): number {
-  const now = new Date();
-  return new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime();
-}
-
-export function formatDate(timestamp: number): string {
-  return new Date(timestamp).toISOString().split("T")[0] ?? "";
-}
-
 export function formatTime(timestamp: number): string {
-  return new Date(timestamp).toISOString().split("T")[1]?.slice(0, 8) ?? "";
+  const d = new Date(timestamp);
+  const p = (n: number) => String(n).padStart(2, "0");
+  return `${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
 }
 
 export function formatNumber(n: number): string {
@@ -131,100 +113,6 @@ export function formatNumber(n: number): string {
   return n.toString();
 }
 
-export function getDayStats(entries: UsageTrackingEntry[]): Map<string, DayStats> {
-  const days = new Map<string, DayStats>();
-  for (const entry of entries) {
-    const date = formatDate(entry.timestamp);
-    const existing = days.get(date) || { date, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0 };
-    days.set(date, {
-      ...existing,
-      requests: existing.requests + 1,
-      satsCost: existing.satsCost + entry.satsCost,
-      promptTokens: existing.promptTokens + entry.promptTokens,
-      completionTokens: existing.completionTokens + entry.completionTokens,
-      totalTokens: existing.totalTokens + entry.totalTokens,
-    });
-  }
-  return days;
-}
-
-export function getHourlyToday(entries: UsageTrackingEntry[]): Map<number, DayStats> {
-  const todayStart = getTodayStart();
-  const hours = new Map<number, DayStats>();
-  for (const entry of entries) {
-    if (entry.timestamp < todayStart) continue;
-    const hour = new Date(entry.timestamp).getHours();
-    const existing = hours.get(hour) || {
-      date: formatDate(entry.timestamp), requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0,
-    };
-    hours.set(hour, {
-      ...existing,
-      requests: existing.requests + 1,
-      satsCost: existing.satsCost + entry.satsCost,
-      promptTokens: existing.promptTokens + entry.promptTokens,
-      completionTokens: existing.completionTokens + entry.completionTokens,
-      totalTokens: existing.totalTokens + entry.totalTokens,
-    });
-  }
-  return hours;
-}
-
-export function getModelStats(entries: UsageTrackingEntry[]): ModelStats[] {
-  const models = new Map<string, ModelStats>();
-  for (const entry of entries) {
-    const existing = models.get(entry.modelId) || {
-      modelId: entry.modelId, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0,
-    };
-    models.set(entry.modelId, {
-      ...existing,
-      requests: existing.requests + 1,
-      satsCost: existing.satsCost + entry.satsCost,
-      promptTokens: existing.promptTokens + entry.promptTokens,
-      completionTokens: existing.completionTokens + entry.completionTokens,
-      totalTokens: existing.totalTokens + entry.totalTokens,
-    });
-  }
-  return Array.from(models.values()).sort((a, b) => b.satsCost - a.satsCost);
-}
-
-export function getProviderStats(entries: UsageTrackingEntry[]): ProviderStats[] {
-  const providers = new Map<string, ProviderStats>();
-  for (const entry of entries) {
-    const url = entry.baseUrl || "unknown";
-    const existing = providers.get(url) || {
-      baseUrl: url, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0,
-    };
-    providers.set(url, {
-      ...existing,
-      requests: existing.requests + 1,
-      satsCost: existing.satsCost + entry.satsCost,
-      promptTokens: existing.promptTokens + entry.promptTokens,
-      completionTokens: existing.completionTokens + entry.completionTokens,
-      totalTokens: existing.totalTokens + entry.totalTokens,
-    });
-  }
-  return Array.from(providers.values()).sort((a, b) => b.satsCost - a.satsCost);
-}
-
-export function getClientStats(entries: UsageTrackingEntry[]): ClientStats[] {
-  const clients = new Map<string, ClientStats>();
-  for (const entry of entries) {
-    const client = entry.client || "unknown";
-    const existing = clients.get(client) || {
-      client, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0,
-    };
-    clients.set(client, {
-      ...existing,
-      requests: existing.requests + 1,
-      satsCost: existing.satsCost + entry.satsCost,
-      promptTokens: existing.promptTokens + entry.promptTokens,
-      completionTokens: existing.completionTokens + entry.completionTokens,
-      totalTokens: existing.totalTokens + entry.totalTokens,
-    });
-  }
-  return Array.from(clients.values()).sort((a, b) => b.satsCost - a.satsCost);
-}
-
 // ─── Client / Npub helpers ────────────────────────────────────────────
 
 export interface ClientInfo {
@@ -258,56 +146,3 @@ export async function fetchClients(): Promise<ClientInfo[]> {
 export function hasAnyNpubs(clients: ClientInfo[]): boolean {
   return clients.some((c) => !!c.ownerNpub);
 }
-
-export function getNpubStats(entries: UsageTrackingEntry[], clients: ClientInfo[]): NpubStats[] {
-  // Build client-id → ownerNpub lookup
-  const clientNpubMap = new Map<string, string>();
-  for (const c of clients) {
-    if (c.ownerNpub) {
-      clientNpubMap.set(c.clientId, c.ownerNpub);
-    }
-  }
-
-  // Aggregate usage per npub
-  const npubs = new Map<string, NpubStats>();
-  for (const entry of entries) {
-    const npub = clientNpubMap.get(entry.client || "");
-    if (!npub) continue; // skip entries whose client has no ownerNpub
-
-    const existing = npubs.get(npub) || {
-      npub,
-      requests: 0,
-      satsCost: 0,
-      promptTokens: 0,
-      completionTokens: 0,
-      totalTokens: 0,
-    };
-    npubs.set(npub, {
-      ...existing,
-      requests: existing.requests + 1,
-      satsCost: existing.satsCost + entry.satsCost,
-      promptTokens: existing.promptTokens + entry.promptTokens,
-      completionTokens: existing.completionTokens + entry.completionTokens,
-      totalTokens: existing.totalTokens + entry.totalTokens,
-    });
-  }
-
-  return Array.from(npubs.values()).sort((a, b) => b.satsCost - a.satsCost);
-}
-
-// ─── Totals ───────────────────────────────────────────────────────────
-
-export function getTotals(entries: UsageTrackingEntry[] | undefined) {
-  if (!entries || !Array.isArray(entries)) {
-    return { promptTokens: 0, completionTokens: 0, totalTokens: 0, satsCost: 0 };
-  }
-  return entries.reduce(
-    (acc, entry) => ({
-      promptTokens: acc.promptTokens + entry.promptTokens,
-      completionTokens: acc.completionTokens + entry.completionTokens,
-      totalTokens: acc.totalTokens + entry.totalTokens,
-      satsCost: acc.satsCost + entry.satsCost,
-    }),
-    { promptTokens: 0, completionTokens: 0, totalTokens: 0, satsCost: 0 }
-  );
-}