weflow-cli 1.6.4 → 1.8.0

package/src/services/messageQuery.ts

@@ -36,3 +36,100 @@ export async function collectMessagesInRange(
 
   return result
 }
+
+/** Why the paging loop stopped. The first two are expected; the last two are not. */
+export type StopReason =
+  /** Reached the requested `from` bound. */
+  | 'range'
+  /** A page came back empty - there is nothing older. */
+  | 'exhausted'
+  /** Collected the caller's limit. */
+  | 'limit'
+  /** A page shorter than pageSize ended the loop before `from` was reached. */
+  | 'short-page'
+
+export interface DetailedMessageCollection {
+  messages: Message[]
+  pages: number
+  stoppedBy: StopReason
+  mayHaveMore: boolean
+  /**
+   * A short page ended the loop while the requested `from` was still ahead
+   * of us.
+   *
+   * `collectMessagesInRange` breaks on a short page, which is right when the
+   * page is short because the conversation ended - but a live database can
+   * also return a short page while rows are being written, because offset
+   * paging shifts underneath. The two are indistinguishable from the result
+   * alone, so this records the difference instead of assuming the benign one.
+   */
+  truncatedEarly: boolean
+}
+
+/**
+ * `collectMessagesInRange` plus the reason it stopped.
+ *
+ * Kept separate rather than folded into the original: that one is covered by
+ * existing tests and used by every export path, and changing what it returns
+ * would change those callers.
+ */
+export async function collectMessagesInRangeDetailed(
+  fetchPage: (limit: number, offset: number) => Promise<Message[]>,
+  limit: number,
+  range: MessageTimeRange,
+  pageSize = 500,
+): Promise<DetailedMessageCollection> {
+  const messages: Message[] = []
+  let offset = 0
+  let pages = 0
+  let stoppedBy: StopReason = 'exhausted'
+  let reachedFrom = range.from === undefined
+
+  while (true) {
+    const page = await fetchPage(pageSize, offset)
+    pages += 1
+    if (page.length === 0) {
+      stoppedBy = 'exhausted'
+      reachedFrom = true
+      break
+    }
+
+    let oldestTimestamp = Number.POSITIVE_INFINITY
+    for (const message of page) {
+      const timestamp = Number(message.createTime)
+      if (!Number.isFinite(timestamp)) continue
+      oldestTimestamp = Math.min(oldestTimestamp, timestamp)
+      if (range.from !== undefined && timestamp < range.from) continue
+      if (range.to !== undefined && timestamp > range.to) continue
+      messages.push(message)
+      if (limit > 0 && messages.length >= limit) {
+        return { messages, pages, stoppedBy: 'limit', mayHaveMore: true, truncatedEarly: false }
+      }
+    }
+
+    offset += page.length
+    if (range.from !== undefined && oldestTimestamp < range.from) {
+      stoppedBy = 'range'
+      reachedFrom = true
+      break
+    }
+    if (page.length < pageSize) {
+      // A short *first* page is unambiguous: nothing has shifted yet, so the
+      // conversation simply has fewer than pageSize messages. A short later
+      // page is the ambiguous one offset paging can produce on a live
+      // database, so it keeps its own name.
+      stoppedBy = pages === 1 ? 'exhausted' : 'short-page'
+      break
+    }
+  }
+
+  return {
+    messages,
+    pages,
+    stoppedBy,
+    // A limit hit is handled above; anything else that is not 'range' may
+    // still have older matching rows we never looked at.
+    mayHaveMore: stoppedBy !== 'range' && stoppedBy !== 'exhausted',
+    truncatedEarly: stoppedBy === 'short-page' && !reachedFrom,
+  }
+}

package/src/services/pythonBridge.ts

@@ -0,0 +1,139 @@
+/**
+ * 调本仓 Python 脚本的**唯一入口**：拿 JSON 回来，或者把失败说清楚。
+ *
+ * 为什么要有这一层：此前有两份实现——`get_todos` 用 `execFile`（参数走 argv），路由用
+ * `spawn`（请求走 stdin）。两份各有一套超时、错误、stdout 解析的写法，正是"同一件事两处
+ * 实现"的老毛病；而工具只会越加越多。
+ *
+ * 三条约定：
+ * 1. **只有 JSON 出口**：脚本要么给出可解析的 JSON，要么给出错误——不解析人类可读文本
+ *    （那种解析在脚本改一行格式时就悄悄失效）。
+ * 2. **JSON 可以在最后一行**：有些脚本（如 route_cards）前面会打印给人看的进度行。
+ *    所以先试整份 stdout，再退一步取"最后一行能解析成 JSON 的"。
+ * 3. **失败要说清是什么失败**：退出码、还是没有 JSON、还是超时——三者分开报，别都变成
+ *    "执行失败"。
+ */
+import { spawn } from 'node:child_process'
+import { join } from 'node:path'
+import { resolvePackageRoot } from '../utils/packageRoot.js'
+import { createPythonProcessEnv } from '../utils/pythonProcessEnv.js'
+import { getPythonCommand } from '../utils/python.js'
+
+export interface ScriptRun {
+  stdout: string
+  stderr: string
+  /** 进程退出码；被信号杀死时为 null */
+  code: number | null
+}
+
+/** 注入点：测试用假 runner 驱动，不 spawn 真进程 */
+export type ScriptRunner = (scriptPath: string, args: string[],
+                            options: { timeoutMs: number; stdin?: string; env?: Record<string, string> }) => Promise<ScriptRun>
+
+export interface JsonResult<T> {
+  ok: boolean
+  data?: T
+  /** 失败原因（已分类：超时 / 退出码 / 没有 JSON） */
+  error?: string
+  /** 失败时的 stderr 尾巴，便于定位 */
+  stderr?: string
+}
+
+const DEFAULT_TIMEOUT_MS = 30_000
+
+function realRunner(scriptPath: string, args: string[],
+                    options: { timeoutMs: number; stdin?: string; env?: Record<string, string> }): Promise<ScriptRun> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(getPythonCommand(), [scriptPath, ...args], {
+      // 用户输入走**环境变量**而不是 argv：这是仓库写进测试的隐私纪律（进程列表里看不到正文）
+      env: createPythonProcessEnv(options.env ?? {}),
+      stdio: ['pipe', 'pipe', 'pipe'],
+    })
+    let stdout = ''
+    let stderr = ''
+    let settled = false
+    const timer = setTimeout(() => {
+      if (settled) return
+      settled = true
+      child.kill()
+      reject(new Error(`脚本超时（${options.timeoutMs}ms）`))
+    }, options.timeoutMs)
+
+    child.stdout.on('data', (chunk: Buffer) => { stdout += chunk.toString() })
+    child.stderr.on('data', (chunk: Buffer) => { stderr += chunk.toString() })
+    child.on('error', (error) => {
+      if (settled) return
+      settled = true
+      clearTimeout(timer)
+      reject(error)
+    })
+    child.on('close', (code) => {
+      if (settled) return
+      settled = true
+      clearTimeout(timer)
+      resolve({ stdout, stderr, code })
+    })
+    if (options.stdin !== undefined) child.stdin.end(options.stdin)
+    else child.stdin.end()
+  })
+}
+
+let runner: ScriptRunner = realRunner
+
+/** 换掉 runner（测试用）。传 null 恢复真实实现。 */
+export function setScriptRunner(next: ScriptRunner | null): void {
+  runner = next ?? realRunner
+}
+
+/** 脚本路径：一律相对包根下的 `scripts/`，不给调用方自己拼路径的自由 */
+export function scriptPath(name: string): string {
+  return join(resolvePackageRoot(import.meta.url), 'scripts', name)
+}
+
+/** 从 stdout 里取 JSON：先试整份，再取最后一行能解析的 */
+export function parseJsonFrom(stdout: string): { ok: boolean; data?: any } {
+  const whole = stdout.trim()
+  if (whole) {
+    try { return { ok: true, data: JSON.parse(whole) } } catch { /* 落下去取最后一行 */ }
+  }
+  const lines = stdout.split(/\r?\n/).map(l => l.trim()).filter(Boolean)
+  for (let i = lines.length - 1; i >= 0; i--) {
+    if (!lines[i].startsWith('{') && !lines[i].startsWith('[')) continue
+    try { return { ok: true, data: JSON.parse(lines[i]) } } catch { /* 继续往前找 */ }
+  }
+  return { ok: false }
+}
+
+/**
+ * 跑一个脚本并解析它给的 JSON。
+ *
+ * **不抛异常**：调用方（工具分支）要的是"这次调用成不成功"，而不是一层 try/catch。
+ * 失败的三种情形分开报——超时、退出码非 0、退出码 0 但没有 JSON。
+ */
+export async function runPythonJson<T = any>(
+  script: string, args: string[],
+  options: { timeoutMs?: number; stdin?: string; env?: Record<string, string> } = {},
+): Promise<JsonResult<T>> {
+  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS
+  let run: ScriptRun
+  try {
+    run = await runner(scriptPath(script), args, { timeoutMs, stdin: options.stdin, env: options.env })
+  } catch (error: any) {
+    return { ok: false, error: String(error?.message ?? error).slice(0, 200) }
+  }
+
+  const parsed = parseJsonFrom(run.stdout)
+  const stderrTail = run.stderr.trim().split(/\r?\n/).filter(Boolean).slice(-3).join(' | ').slice(0, 300)
+
+  if (run.code !== 0) {
+    return { ok: false, error: `脚本退出码 ${run.code}`, stderr: stderrTail }
+  }
+  if (!parsed.ok) {
+    return { ok: false, error: '脚本没有给出可解析的 JSON', stderr: stderrTail }
+  }
+  const data: any = parsed.data
+  if (data && typeof data === 'object' && 'success' in data && data.success === false) {
+    return { ok: false, error: String(data.error ?? '脚本报错').slice(0, 200), stderr: stderrTail }
+  }
+  return { ok: true, data: data as T }
+}

package/src/services/syncService.ts

@@ -0,0 +1,363 @@
+/**
+ * One sync pass: pick a window, read it, dedupe, decide coverage, record state.
+ *
+ * Deliberately not a general job runner - it is the single caller of the sync
+ * state contract for now (docs/SYNC_CONTRACT.md, D-029).
+ */
+import type { Message } from '../types.js'
+import type { ShardReport } from '../core/ntCore.js'
+import {
+  DEFAULT_OVERLAP_SECONDS,
+  SYNC_SOURCE,
+  emptySyncState,
+  makeJobId,
+  syncStateStore,
+  type JobRecord,
+  type SyncCoverage,
+  type SyncState,
+  type SyncStateStore,
+} from './syncState.js'
+
+/** A window wider than this is worth warning about: the read is not bounded. */
+export const WIDE_WINDOW_SECONDS = 90 * 24 * 3600
+
+export class SyncRangeRequiredError extends Error {
+  readonly code = 'SYNC_RANGE_REQUIRED'
+  constructor() {
+    super('首次同步需要 --since <日期> 或 --full')
+    this.name = 'SyncRangeRequiredError'
+  }
+}
+
+/**
+ * Stable identity for one message.
+ *
+ * `serverId` wins when nonzero; otherwise `localId` is paired with
+ * `createTime`. A bare `localId` is never an identity - it restarts in every
+ * shard, which is the whole reason D-014 exists.
+ *
+ * `shard` is intentionally absent: messages do not carry it, adding one would
+ * touch the fenced export contract, and overlap dedup does not need it (the
+ * same message read twice yields the same serverId or the same pair).
+ */
+export function messageIdentity(source: string, talker: string, message: Message): string {
+  const serverId = String(message.serverId ?? '').trim()
+  if (serverId && serverId !== '0') return `${source}|${talker}|s:${serverId}`
+  return `${source}|${talker}|l:${message.localId}|t:${message.createTime}`
+}
+
+export interface WindowOptions {
+  since?: number
+  full?: boolean
+  overlapSeconds?: number
+  now?: number
+}
+
+export interface SyncWindow {
+  from: number | null
+  to: number
+  overlapSeconds: number
+  full: boolean
+}
+
+/**
+ * Work out what to read.
+ *
+ * Without a prior checkpoint there is nothing to resume from, so the caller
+ * must say how far back to go - guessing "everything" would silently turn a
+ * routine sync into a full-history read.
+ */
+export function computeWindow(prior: SyncState | null, options: WindowOptions): SyncWindow {
+  const overlapSeconds = options.overlapSeconds ?? prior?.checkpoint.overlapSeconds
+    ?? DEFAULT_OVERLAP_SECONDS
+  const to = options.now ?? Math.floor(Date.now() / 1000)
+
+  if (options.full) {
+    return { from: null, to, overlapSeconds, full: true }
+  }
+  if (options.since !== undefined) {
+    return { from: options.since, to, overlapSeconds, full: false }
+  }
+  const newest = prior?.checkpoint.newestCreateTime
+  if (prior && typeof newest === 'number') {
+    // Back up by the overlap: a second-resolution timestamp is not a cursor,
+    // so the boundary messages have to be read again and deduplicated.
+    return { from: Math.max(0, newest - overlapSeconds), to, overlapSeconds, full: false }
+  }
+  throw new SyncRangeRequiredError()
+}
+
+/** Drop repeats the overlap window re-read. */
+export function dedupeMessages(
+  messages: Message[], source: string, talker: string,
+): { unique: Message[]; duplicateCount: number } {
+  const seen = new Set<string>()
+  const unique: Message[] = []
+  for (const message of messages) {
+    const identity = messageIdentity(source, talker, message)
+    if (seen.has(identity)) continue
+    seen.add(identity)
+    unique.push(message)
+  }
+  return { unique, duplicateCount: messages.length - unique.length }
+}
+
+/**
+ * Decide how much of the read we can stand behind.
+ *
+ * `partial` is the only value that must not be reported as success.
+ *
+ * A short-page truncation cannot occur here because this pass reads in one
+ * call rather than paging; `collectMessagesInRangeDetailed` detects that case
+ * for the paged readers and is where a future revision of this should look.
+ */
+export function deriveCoverage(input: {
+  shards?: ShardReport
+  mayHaveMore: boolean
+}): SyncCoverage {
+  if (input.mayHaveMore) return 'partial'
+  // Any per-shard reason means those rows were not read: a fault after opening
+  // (READ_FAILED), a table the reader can name nothing in (SCHEMA_MISMATCH), or
+  // a window the shard cannot express (WINDOW_UNAVAILABLE). Coverage is about
+  // whether the range was read, so all of them make it partial. A shard read
+  // with columns missing is *not* one of these - its rows did come back.
+  // A shard that never opened is a different answer and stays 'unverified'
+  // below: nothing is known about its rows either way, which is not the same
+  // claim as knowing they were missed.
+  const anyShardUnread = input.shards?.items.some(item => item.opened && item.reason)
+  if (anyShardUnread) return 'partial'
+  if (!input.shards) return 'unverified'
+  // A shard that never opened is not knowable either way; say so rather than
+  // claiming completeness or declaring failure.
+  if (input.shards.failed > 0) return 'unverified'
+  return 'complete'
+}
+
+export function buildNextState(input: {
+  prior: SyncState | null
+  talker: string
+  scope: string
+  source: string
+  window: SyncWindow
+  recordsRead: number
+  recordsDeduplicated: number
+  recordsReturned: number
+  newestCreateTime: number | null
+  oldestCreateTime: number | null
+  shards?: ShardReport
+  coverage: SyncCoverage
+  mayHaveMore: boolean
+  warnings: string[]
+  now?: number
+}): SyncState {
+  const state = input.prior ?? emptySyncState(input.talker, input.scope, input.source,
+    input.window.overlapSeconds)
+  // Same form as coveredFrom/coveredTo: an offset, not a bare Z. A file that
+  // mixes the two makes "which local day was this?" unanswerable.
+  const stamp = isoFromSeconds(input.now ?? Math.floor(Date.now() / 1000))
+
+  const priorFrom = state.coveredFrom
+  let coveredFrom: string | null = priorFrom
+  if (input.window.full) {
+    // The database's true minimum is not knowable without another query, so
+    // this is the oldest row we actually saw and says so in `warnings`.
+    coveredFrom = input.oldestCreateTime ? isoFromSeconds(input.oldestCreateTime) : priorFrom
+  } else if (input.window.from !== null) {
+    const candidate = isoFromSeconds(input.window.from)
+    coveredFrom = priorFrom && priorFrom < candidate ? priorFrom : candidate
+  }
+
+  return {
+    ...state,
+    scope: input.scope,
+    talker: input.talker,
+    source: input.source,
+    coveredFrom,
+    coveredTo: isoFromSeconds(input.window.to),
+    checkpoint: {
+      newestCreateTime: input.newestCreateTime ?? state.checkpoint.newestCreateTime,
+      overlapSeconds: input.window.overlapSeconds,
+    },
+    recordsRead: input.recordsRead,
+    recordsDeduplicated: input.recordsDeduplicated,
+    recordsReturned: input.recordsReturned,
+    shardsRead: input.shards?.opened ?? 0,
+    shardsFailed: input.shards?.failed ?? 0,
+    shards: input.shards
+      ? { scanned: input.shards.scanned, opened: input.shards.opened,
+          failed: input.shards.failed, items: input.shards.items }
+      : { scanned: 0, opened: 0, failed: 0, items: [] },
+    coverage: input.coverage,
+    mayHaveMore: input.mayHaveMore,
+    warnings: input.warnings,
+    // The attempt always advances; the success does not, so a partial run is
+    // never mistaken for a good one later.
+    lastAttempt: stamp,
+    lastSuccessfulRun: input.coverage === 'partial' ? state.lastSuccessfulRun : stamp,
+    updatedAt: stamp,
+  }
+}
+
+function isoFromSeconds(seconds: number): string {
+  const date = new Date(seconds * 1000)
+  const offsetMinutes = -date.getTimezoneOffset()
+  const sign = offsetMinutes >= 0 ? '+' : '-'
+  const pad = (value: number) => String(Math.floor(Math.abs(value))).padStart(2, '0')
+  const local = new Date(date.getTime() + offsetMinutes * 60_000)
+  const body = local.toISOString().slice(0, 19)
+  return `${body}${sign}${pad(offsetMinutes / 60)}:${pad(offsetMinutes % 60)}`
+}
+
+export interface SyncRunOptions extends WindowOptions {
+  source?: string
+  scope?: string
+  limit?: number
+  /**
+   * Injected for tests; the CLI wires this to chatService.
+   *
+   * `from` is a hint that a backend may push into SQL. It is never a
+   * correctness guarantee: runSync still applies the window itself, because a
+   * backend that cannot honour the lower bound reads everything instead.
+   */
+  read: (
+    talker: string,
+    limit: number,
+    from: number | null
+  ) => Promise<{ messages: Message[]; shards?: ShardReport }>
+  write?: boolean
+  /** Defaults to the real store; tests pass one rooted in a temp directory. */
+  store?: SyncStateStore
+}
+
+export interface SyncRunResult {
+  success: boolean
+  code?: string
+  action: 'sync.messages'
+  scope: string
+  window: SyncWindow
+  recordsRead: number
+  recordsDeduplicated: number
+  recordsReturned: number
+  shards?: ShardReport
+  coverage: SyncCoverage
+  mayHaveMore: boolean
+  partial: boolean
+  warnings: string[]
+  jobId: string | null
+  state: SyncState | null
+}
+
+/**
+ * Run one pass. Writes state and a job record unless `write: false`.
+ *
+ * A partial run still writes (the spec forbids discarding usable results) but
+ * returns `success: false` with `code: 'SYNC_PARTIAL'` so nothing downstream
+ * treats it as complete.
+ */
+export async function runSync(talker: string, options: SyncRunOptions): Promise<SyncRunResult> {
+  const source = options.source ?? SYNC_SOURCE
+  const scope = options.scope ?? talker
+  const store = options.store ?? syncStateStore
+  const prior = store.read(source, talker)
+  const window = computeWindow(prior, options)
+
+  const warnings: string[] = []
+  if (window.full) warnings.push('full-scan')
+  if (window.from !== null && window.to - window.from > WIDE_WINDOW_SECONDS) {
+    warnings.push('wide-window')
+  }
+
+  const limit = options.limit ?? 0
+  const read = await options.read(talker, limit, window.from)
+  const { unique, duplicateCount } = dedupeMessages(read.messages, source, talker)
+
+  // The window is applied here as well as being pushed down, and this filter -
+  // not the pushdown - is what makes the result correct: a backend without
+  // range support returns the whole conversation, and one that cannot express
+  // the lower bound says so per shard rather than quietly dropping the bound.
+  const inWindow = window.from === null
+    ? unique
+    : unique.filter(m => Number(m.createTime) >= (window.from as number))
+
+  const newestCreateTime = inWindow.length
+    ? Math.max(...inWindow.map(m => Number(m.createTime) || 0))
+    : null
+  const oldestCreateTime = inWindow.length
+    ? Math.min(...inWindow.map(m => Number(m.createTime) || 0))
+    : null
+
+  const mayHaveMore = limit > 0 && read.messages.length >= limit
+  const coverage = deriveCoverage({ shards: read.shards, mayHaveMore })
+  if (!read.shards) warnings.push('shard-report-unavailable')
+  if (read.shards) {
+    for (const item of read.shards.items) {
+      if (!item.opened) warnings.push(`shard-unverified:${item.name}`)
+      else if (item.reason === 'READ_FAILED') warnings.push(`shard-read-failed:${item.name}`)
+      // A shard the reader could not name anything in, or could not apply the
+      // window to. Both mean "this range was not read here", which must not
+      // pass as covered - hence a warning rather than a silent skip.
+      else if (item.reason) warnings.push(`shard-not-read:${item.name}:${item.reason}`)
+      // Read, but with fields missing. Column names are schema, not content,
+      // so this stays safe to record in the state file.
+      else if (item.missingColumns?.length) {
+        warnings.push(`shard-columns-missing:${item.name}:${item.missingColumns.join('+')}`)
+      }
+    }
+  }
+
+  const next = buildNextState({
+    prior, talker, scope, source, window,
+    recordsRead: read.messages.length,
+    recordsDeduplicated: duplicateCount,
+    recordsReturned: inWindow.length,
+    newestCreateTime, oldestCreateTime,
+    shards: read.shards, coverage, mayHaveMore, warnings,
+    // Same clock as the window: without this the recorded attempt would come
+    // from Date.now() while window.to came from the caller's `now`.
+    now: options.now,
+  })
+
+  const partial = coverage === 'partial'
+  let jobId: string | null = null
+  if (options.write !== false) {
+    jobId = makeJobId()
+    const job: JobRecord = {
+      schema: 'weflow-job/v1',
+      jobId,
+      kind: 'sync.messages',
+      state: partial ? 'partial' : 'completed',
+      createdAt: next.lastAttempt as string,
+      updatedAt: next.lastAttempt as string,
+      window: { from: window.from, to: window.to },
+      progress: { completed: inWindow.length, total: read.messages.length },
+      resumeable: partial,
+      result: {
+        records: inWindow.length,
+        recordsDeduplicated: duplicateCount,
+        shardsFailed: read.shards?.failed ?? 0,
+        coverage,
+      },
+      error: partial ? 'SYNC_PARTIAL' : null,
+    }
+    store.write(next)
+    store.writeJob(job)
+  }
+
+  return {
+    success: !partial,
+    code: partial ? 'SYNC_PARTIAL' : undefined,
+    action: 'sync.messages',
+    scope,
+    window,
+    recordsRead: read.messages.length,
+    recordsDeduplicated: duplicateCount,
+    recordsReturned: inWindow.length,
+    shards: read.shards,
+    coverage,
+    mayHaveMore,
+    partial,
+    warnings,
+    jobId,
+    state: next,
+  }
+}