weflow-cli 1.6.3 → 1.7.0

package/src/services/syncService.ts

@@ -0,0 +1,335 @@
+/**
+ * One sync pass: pick a window, read it, dedupe, decide coverage, record state.
+ *
+ * Deliberately not a general job runner - it is the single caller of the sync
+ * state contract for now (docs/SYNC_CONTRACT.md, D-029).
+ */
+import type { Message } from '../types.js'
+import type { ShardReport } from '../core/ntCore.js'
+import {
+  DEFAULT_OVERLAP_SECONDS,
+  SYNC_SOURCE,
+  emptySyncState,
+  makeJobId,
+  syncStateStore,
+  type JobRecord,
+  type SyncCoverage,
+  type SyncState,
+  type SyncStateStore,
+} from './syncState.js'
+
+/** A window wider than this is worth warning about: the read is not bounded. */
+export const WIDE_WINDOW_SECONDS = 90 * 24 * 3600
+
+export class SyncRangeRequiredError extends Error {
+  readonly code = 'SYNC_RANGE_REQUIRED'
+  constructor() {
+    super('首次同步需要 --since <日期> 或 --full')
+    this.name = 'SyncRangeRequiredError'
+  }
+}
+
+/**
+ * Stable identity for one message.
+ *
+ * `serverId` wins when nonzero; otherwise `localId` is paired with
+ * `createTime`. A bare `localId` is never an identity - it restarts in every
+ * shard, which is the whole reason D-014 exists.
+ *
+ * `shard` is intentionally absent: messages do not carry it, adding one would
+ * touch the fenced export contract, and overlap dedup does not need it (the
+ * same message read twice yields the same serverId or the same pair).
+ */
+export function messageIdentity(source: string, talker: string, message: Message): string {
+  const serverId = String(message.serverId ?? '').trim()
+  if (serverId && serverId !== '0') return `${source}|${talker}|s:${serverId}`
+  return `${source}|${talker}|l:${message.localId}|t:${message.createTime}`
+}
+
+export interface WindowOptions {
+  since?: number
+  full?: boolean
+  overlapSeconds?: number
+  now?: number
+}
+
+export interface SyncWindow {
+  from: number | null
+  to: number
+  overlapSeconds: number
+  full: boolean
+}
+
+/**
+ * Work out what to read.
+ *
+ * Without a prior checkpoint there is nothing to resume from, so the caller
+ * must say how far back to go - guessing "everything" would silently turn a
+ * routine sync into a full-history read.
+ */
+export function computeWindow(prior: SyncState | null, options: WindowOptions): SyncWindow {
+  const overlapSeconds = options.overlapSeconds ?? prior?.checkpoint.overlapSeconds
+    ?? DEFAULT_OVERLAP_SECONDS
+  const to = options.now ?? Math.floor(Date.now() / 1000)
+
+  if (options.full) {
+    return { from: null, to, overlapSeconds, full: true }
+  }
+  if (options.since !== undefined) {
+    return { from: options.since, to, overlapSeconds, full: false }
+  }
+  const newest = prior?.checkpoint.newestCreateTime
+  if (prior && typeof newest === 'number') {
+    // Back up by the overlap: a second-resolution timestamp is not a cursor,
+    // so the boundary messages have to be read again and deduplicated.
+    return { from: Math.max(0, newest - overlapSeconds), to, overlapSeconds, full: false }
+  }
+  throw new SyncRangeRequiredError()
+}
+
+/** Drop repeats the overlap window re-read. */
+export function dedupeMessages(
+  messages: Message[], source: string, talker: string,
+): { unique: Message[]; duplicateCount: number } {
+  const seen = new Set<string>()
+  const unique: Message[] = []
+  for (const message of messages) {
+    const identity = messageIdentity(source, talker, message)
+    if (seen.has(identity)) continue
+    seen.add(identity)
+    unique.push(message)
+  }
+  return { unique, duplicateCount: messages.length - unique.length }
+}
+
+/**
+ * Decide how much of the read we can stand behind.
+ *
+ * `partial` is the only value that must not be reported as success.
+ *
+ * A short-page truncation cannot occur here because this pass reads in one
+ * call rather than paging; `collectMessagesInRangeDetailed` detects that case
+ * for the paged readers and is where a future revision of this should look.
+ */
+export function deriveCoverage(input: {
+  shards?: ShardReport
+  mayHaveMore: boolean
+}): SyncCoverage {
+  if (input.mayHaveMore) return 'partial'
+  // A shard that faulted *after* opening means its rows are only partly read.
+  const readFailed = input.shards?.items.some(item => item.reason === 'READ_FAILED')
+  if (readFailed) return 'partial'
+  if (!input.shards) return 'unverified'
+  // A shard that never opened is not knowable either way; say so rather than
+  // claiming completeness or declaring failure.
+  if (input.shards.failed > 0) return 'unverified'
+  return 'complete'
+}
+
+export function buildNextState(input: {
+  prior: SyncState | null
+  talker: string
+  scope: string
+  source: string
+  window: SyncWindow
+  recordsRead: number
+  recordsDeduplicated: number
+  recordsReturned: number
+  newestCreateTime: number | null
+  oldestCreateTime: number | null
+  shards?: ShardReport
+  coverage: SyncCoverage
+  mayHaveMore: boolean
+  warnings: string[]
+  now?: number
+}): SyncState {
+  const state = input.prior ?? emptySyncState(input.talker, input.scope, input.source,
+    input.window.overlapSeconds)
+  // Same form as coveredFrom/coveredTo: an offset, not a bare Z. A file that
+  // mixes the two makes "which local day was this?" unanswerable.
+  const stamp = isoFromSeconds(input.now ?? Math.floor(Date.now() / 1000))
+
+  const priorFrom = state.coveredFrom
+  let coveredFrom: string | null = priorFrom
+  if (input.window.full) {
+    // The database's true minimum is not knowable without another query, so
+    // this is the oldest row we actually saw and says so in `warnings`.
+    coveredFrom = input.oldestCreateTime ? isoFromSeconds(input.oldestCreateTime) : priorFrom
+  } else if (input.window.from !== null) {
+    const candidate = isoFromSeconds(input.window.from)
+    coveredFrom = priorFrom && priorFrom < candidate ? priorFrom : candidate
+  }
+
+  return {
+    ...state,
+    scope: input.scope,
+    talker: input.talker,
+    source: input.source,
+    coveredFrom,
+    coveredTo: isoFromSeconds(input.window.to),
+    checkpoint: {
+      newestCreateTime: input.newestCreateTime ?? state.checkpoint.newestCreateTime,
+      overlapSeconds: input.window.overlapSeconds,
+    },
+    recordsRead: input.recordsRead,
+    recordsDeduplicated: input.recordsDeduplicated,
+    recordsReturned: input.recordsReturned,
+    shardsRead: input.shards?.opened ?? 0,
+    shardsFailed: input.shards?.failed ?? 0,
+    shards: input.shards
+      ? { scanned: input.shards.scanned, opened: input.shards.opened,
+          failed: input.shards.failed, items: input.shards.items }
+      : { scanned: 0, opened: 0, failed: 0, items: [] },
+    coverage: input.coverage,
+    mayHaveMore: input.mayHaveMore,
+    warnings: input.warnings,
+    // The attempt always advances; the success does not, so a partial run is
+    // never mistaken for a good one later.
+    lastAttempt: stamp,
+    lastSuccessfulRun: input.coverage === 'partial' ? state.lastSuccessfulRun : stamp,
+    updatedAt: stamp,
+  }
+}
+
+function isoFromSeconds(seconds: number): string {
+  const date = new Date(seconds * 1000)
+  const offsetMinutes = -date.getTimezoneOffset()
+  const sign = offsetMinutes >= 0 ? '+' : '-'
+  const pad = (value: number) => String(Math.floor(Math.abs(value))).padStart(2, '0')
+  const local = new Date(date.getTime() + offsetMinutes * 60_000)
+  const body = local.toISOString().slice(0, 19)
+  return `${body}${sign}${pad(offsetMinutes / 60)}:${pad(offsetMinutes % 60)}`
+}
+
+export interface SyncRunOptions extends WindowOptions {
+  source?: string
+  scope?: string
+  limit?: number
+  /** Injected for tests; the CLI wires this to chatService. */
+  read: (talker: string, limit: number) => Promise<{ messages: Message[]; shards?: ShardReport }>
+  write?: boolean
+  /** Defaults to the real store; tests pass one rooted in a temp directory. */
+  store?: SyncStateStore
+}
+
+export interface SyncRunResult {
+  success: boolean
+  code?: string
+  action: 'sync.messages'
+  scope: string
+  window: SyncWindow
+  recordsRead: number
+  recordsDeduplicated: number
+  recordsReturned: number
+  shards?: ShardReport
+  coverage: SyncCoverage
+  mayHaveMore: boolean
+  partial: boolean
+  warnings: string[]
+  jobId: string | null
+  state: SyncState | null
+}
+
+/**
+ * Run one pass. Writes state and a job record unless `write: false`.
+ *
+ * A partial run still writes (the spec forbids discarding usable results) but
+ * returns `success: false` with `code: 'SYNC_PARTIAL'` so nothing downstream
+ * treats it as complete.
+ */
+export async function runSync(talker: string, options: SyncRunOptions): Promise<SyncRunResult> {
+  const source = options.source ?? SYNC_SOURCE
+  const scope = options.scope ?? talker
+  const store = options.store ?? syncStateStore
+  const prior = store.read(source, talker)
+  const window = computeWindow(prior, options)
+
+  const warnings: string[] = []
+  if (window.full) warnings.push('full-scan')
+  if (window.from !== null && window.to - window.from > WIDE_WINDOW_SECONDS) {
+    warnings.push('wide-window')
+  }
+
+  const limit = options.limit ?? 0
+  const read = await options.read(talker, limit)
+  const { unique, duplicateCount } = dedupeMessages(read.messages, source, talker)
+
+  // Without a native range pushdown the read is the whole conversation, so the
+  // window is applied here. This is why the sync is correct but not cheap yet.
+  const inWindow = window.from === null
+    ? unique
+    : unique.filter(m => Number(m.createTime) >= (window.from as number))
+
+  const newestCreateTime = inWindow.length
+    ? Math.max(...inWindow.map(m => Number(m.createTime) || 0))
+    : null
+  const oldestCreateTime = inWindow.length
+    ? Math.min(...inWindow.map(m => Number(m.createTime) || 0))
+    : null
+
+  const mayHaveMore = limit > 0 && read.messages.length >= limit
+  const coverage = deriveCoverage({ shards: read.shards, mayHaveMore })
+  if (!read.shards) warnings.push('shard-report-unavailable')
+  if (read.shards && read.shards.failed > 0) {
+    for (const item of read.shards.items) {
+      if (!item.opened) warnings.push(`shard-unverified:${item.name}`)
+      else if (item.reason === 'READ_FAILED') warnings.push(`shard-read-failed:${item.name}`)
+    }
+  }
+
+  const next = buildNextState({
+    prior, talker, scope, source, window,
+    recordsRead: read.messages.length,
+    recordsDeduplicated: duplicateCount,
+    recordsReturned: inWindow.length,
+    newestCreateTime, oldestCreateTime,
+    shards: read.shards, coverage, mayHaveMore, warnings,
+    // Same clock as the window: without this the recorded attempt would come
+    // from Date.now() while window.to came from the caller's `now`.
+    now: options.now,
+  })
+
+  const partial = coverage === 'partial'
+  let jobId: string | null = null
+  if (options.write !== false) {
+    jobId = makeJobId()
+    const job: JobRecord = {
+      schema: 'weflow-job/v1',
+      jobId,
+      kind: 'sync.messages',
+      state: partial ? 'partial' : 'completed',
+      createdAt: next.lastAttempt as string,
+      updatedAt: next.lastAttempt as string,
+      window: { from: window.from, to: window.to },
+      progress: { completed: inWindow.length, total: read.messages.length },
+      resumeable: partial,
+      result: {
+        records: inWindow.length,
+        recordsDeduplicated: duplicateCount,
+        shardsFailed: read.shards?.failed ?? 0,
+        coverage,
+      },
+      error: partial ? 'SYNC_PARTIAL' : null,
+    }
+    store.write(next)
+    store.writeJob(job)
+  }
+
+  return {
+    success: !partial,
+    code: partial ? 'SYNC_PARTIAL' : undefined,
+    action: 'sync.messages',
+    scope,
+    window,
+    recordsRead: read.messages.length,
+    recordsDeduplicated: duplicateCount,
+    recordsReturned: inWindow.length,
+    shards: read.shards,
+    coverage,
+    mayHaveMore,
+    partial,
+    warnings,
+    jobId,
+    state: next,
+  }
+}

package/src/services/syncState.ts

@@ -0,0 +1,275 @@
+/**
+ * 同步检查点与任务记录的本地状态。
+ *
+ * 两个位置, 生命周期不同 (见 docs/SYNC_CONTRACT.md):
+ *   ~/.weflow-cli/sync/<source>-<name>-<hash>.json  跨运行的检查点, 每会话一份
+ *   ~/.weflow-cli/jobs/<jobId>.json                 单次运行的记录
+ *
+ * 按会话分文件而不是一个全局文件: 全局文件要求跨会话读改写并合并, 分文件则每次
+ * 写入都是整文件原子替换 (沿用 assistantMemory.ts 的 tmp + rename 做法)。
+ *
+ * 两条本模块特有的规矩:
+ *   - 遇到不认识的 schema 一律拒绝覆盖并报错, 不猜测字段含义;
+ *   - 写盘前统一过一遍脱敏, 保证密钥/绝对路径不会进状态文件 (D-001 / D-002 / D-025)。
+ */
+import { createHash } from 'crypto'
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from 'fs'
+import { join } from 'path'
+import os from 'os'
+
+export const SYNC_SCHEMA = 'weflow-sync/v1'
+export const JOB_SCHEMA = 'weflow-job/v1'
+export const SYNC_SOURCE = 'wechat-nt'
+export const DEFAULT_OVERLAP_SECONDS = 300
+
+/** 单次运行的结局。partial 是本项目新增的一档——部分完成不算成功。 */
+export type JobState = 'planned' | 'waiting_confirmation' | 'running' | 'paused'
+  | 'completed' | 'partial' | 'failed' | 'cancelled'
+
+/**
+ * 覆盖可信度。这是对「有没有读全」的正面回答, 不是内部细节。
+ *
+ *  - complete   扫到的分片全部打开、打开的全读成功, 且没有更多
+ *  - unverified 有分片打不开, 但打开的都读成功——已读部分可信, 未打开的分片内容无法判断
+ *  - partial    已打开的分片读失败, 或还有更多, 或分页提前结束
+ */
+export type SyncCoverage = 'complete' | 'unverified' | 'partial'
+
+export interface SyncShardItem {
+  /** 仅 basename——绝不落绝对路径 */
+  name: string
+  opened: boolean
+  hasTalkerTable: boolean | null
+  rowsForTalker: number | null
+  /** null | KEY_REJECTED | OPEN_FAILED | READ_FAILED */
+  reason: string | null
+}
+
+export interface SyncState {
+  schema: string
+  schemaVersion: number
+  source: string
+  scope: string
+  talker: string
+  lastSuccessfulRun: string | null
+  lastAttempt: string | null
+  coveredFrom: string | null
+  coveredTo: string | null
+  checkpoint: { newestCreateTime: number | null; overlapSeconds: number }
+  recordsRead: number
+  recordsDeduplicated: number
+  recordsReturned: number
+  shardsRead: number
+  shardsFailed: number
+  shards: { scanned: number; opened: number; failed: number; items: SyncShardItem[] }
+  coverage: SyncCoverage
+  mayHaveMore: boolean
+  warnings: string[]
+  updatedAt: string | null
+}
+
+export interface JobRecord {
+  schema: string
+  jobId: string
+  kind: string
+  state: JobState
+  createdAt: string
+  updatedAt: string
+  window: { from: number | null; to: number | null } | null
+  progress: { completed: number; total: number } | null
+  resumeable: boolean
+  result: Record<string, unknown> | null
+  error: string | null
+}
+
+/** 未知 schema 时不覆盖原文件, 交由调用方决定如何处理。 */
+export class UnknownSchemaError extends Error {
+  constructor(public readonly path: string, public readonly found: unknown) {
+    super(`不认识的 schema: ${String(found)} (${path})`)
+    this.name = 'UnknownSchemaError'
+  }
+}
+
+const CONFIG_DIR = join(os.homedir(), '.weflow-cli')
+const SYNC_DIR = join(CONFIG_DIR, 'sync')
+const JOBS_DIR = join(CONFIG_DIR, 'jobs')
+
+const FORBIDDEN_KEYS = /^(ntKey|ntSalt|decryptKey|favKey|favPassphrase|contactKey|snsKey|wechatOcToken|apiKey|password|auth)$/i
+/** 密钥形状的十六进制串 (>=32 hex)。 */
+const HEX_KEY = /\b[0-9a-f]{32,}\b/gi
+/** Windows 盘符、UNC、以及 POSIX 绝对路径。三类都要, 早期只处理了第一类。 */
+const ABS_PATH = /(?:[A-Za-z]:[\\/][^\s"']*|\\\\[^\s"']+|\/(?:Users|home|root|var|tmp|opt|mnt)\/[^\s"']*)/g
+
+/**
+ * 递归剔除不该落盘的字段。
+ *
+ * 状态文件是本机产物, 但一旦被日志、issue 或截图带出去就是泄露, 所以在写入
+ * 这一层兜底, 而不是指望每个调用方都记得。
+ */
+export function sanitizeForState(value: unknown): unknown {
+  if (typeof value === 'string') {
+    return value.replace(HEX_KEY, '<redacted>').replace(ABS_PATH, '<path>')
+  }
+  if (Array.isArray(value)) return value.map(sanitizeForState)
+  if (value && typeof value === 'object') {
+    const out: Record<string, unknown> = {}
+    for (const [key, item] of Object.entries(value)) {
+      if (FORBIDDEN_KEYS.test(key)) continue
+      out[key] = sanitizeForState(item)
+    }
+    return out
+  }
+  return value
+}
+
+/**
+ * 文件名里只放一个可读前缀和一个 hash。
+ *
+ * hash 后缀让 `wxid_a@chatroom` 与 `wxid_a_chatroom` 落到不同文件, 而显示名
+ * 不会出现在目录列表里。
+ */
+export function talkerSlug(talker: string): string {
+  const readable = talker.replace(/[^A-Za-z0-9._-]+/g, '_').slice(0, 60)
+  const digest = createHash('sha256').update(talker).digest('hex').slice(0, 8)
+  return `${readable}-${digest}`
+}
+
+function writeAtomic(path: string, payload: unknown): void {
+  if (!existsSync(CONFIG_DIR)) mkdirSync(CONFIG_DIR, { recursive: true })
+  const tmp = `${path}.tmp`
+  writeFileSync(tmp, JSON.stringify(payload, null, 2), 'utf8')
+  renameSync(tmp, path)
+}
+
+function readJson(path: string): unknown {
+  try {
+    return JSON.parse(readFileSync(path, 'utf8'))
+  } catch {
+    return null
+  }
+}
+
+export function emptySyncState(
+  talker: string,
+  scope = talker,
+  source = SYNC_SOURCE,
+  overlapSeconds = DEFAULT_OVERLAP_SECONDS,
+): SyncState {
+  return {
+    schema: SYNC_SCHEMA,
+    schemaVersion: 1,
+    source,
+    scope,
+    talker,
+    lastSuccessfulRun: null,
+    lastAttempt: null,
+    coveredFrom: null,
+    coveredTo: null,
+    checkpoint: { newestCreateTime: null, overlapSeconds },
+    recordsRead: 0,
+    recordsDeduplicated: 0,
+    recordsReturned: 0,
+    shardsRead: 0,
+    shardsFailed: 0,
+    shards: { scanned: 0, opened: 0, failed: 0, items: [] },
+    coverage: 'unverified',
+    mayHaveMore: false,
+    warnings: [],
+    updatedAt: null,
+  }
+}
+
+export class SyncStateStore {
+  constructor(
+    private readonly syncDir: string = SYNC_DIR,
+    private readonly jobsDir: string = JOBS_DIR,
+  ) {}
+
+  statePath(source: string, talker: string): string {
+    return join(this.syncDir, `${source}-${talkerSlug(talker)}.json`)
+  }
+
+  /** 读取检查点。无记录返回 null; schema 不认识则抛错, 不猜测。 */
+  read(source: string, talker: string): SyncState | null {
+    const path = this.statePath(source, talker)
+    const raw = readJson(path)
+    if (raw === null) return null
+    const state = raw as SyncState
+    if (state.schema !== SYNC_SCHEMA) throw new UnknownSchemaError(path, state.schema)
+    return state
+  }
+
+  write(state: SyncState): void {
+    const path = this.statePath(state.source, state.talker)
+    if (!existsSync(this.syncDir)) mkdirSync(this.syncDir, { recursive: true })
+    writeAtomic(path, sanitizeForState({ ...state, schema: SYNC_SCHEMA }))
+  }
+
+  /** 所有会话的检查点, 最近尝试在前。目录缺失或单个文件损坏都不影响其余。 */
+  list(): SyncState[] {
+    try {
+      if (!existsSync(this.syncDir)) return []
+      const states: SyncState[] = []
+      for (const name of readdirSync(this.syncDir)) {
+        if (!name.endsWith('.json')) continue
+        const raw = readJson(join(this.syncDir, name))
+        if (!raw || (raw as SyncState).schema !== SYNC_SCHEMA) continue
+        states.push(raw as SyncState)
+      }
+      return states.sort((a, b) => String(b.lastAttempt ?? '').localeCompare(String(a.lastAttempt ?? '')))
+    } catch {
+      return []
+    }
+  }
+
+  jobPath(jobId: string): string {
+    return join(this.jobsDir, `${jobId}.json`)
+  }
+
+  writeJob(job: JobRecord): void {
+    if (!existsSync(this.jobsDir)) mkdirSync(this.jobsDir, { recursive: true })
+    writeAtomic(this.jobPath(job.jobId), sanitizeForState({ ...job, schema: JOB_SCHEMA }))
+  }
+
+  readJob(jobId: string): JobRecord | null {
+    const path = this.jobPath(jobId)
+    const raw = readJson(path)
+    if (raw === null) return null
+    const job = raw as JobRecord
+    if (job.schema !== JOB_SCHEMA) throw new UnknownSchemaError(path, job.schema)
+    return job
+  }
+
+  /** 最近的 job 记录, 最新在前。 */
+  recentJobs(limit = 10): JobRecord[] {
+    try {
+      if (!existsSync(this.jobsDir)) return []
+      const ids = readdirSync(this.jobsDir)
+        .filter(name => name.endsWith('.json'))
+        .map(name => name.slice(0, -'.json'.length))
+        .sort()
+        .reverse()
+        .slice(0, limit)
+      const jobs: JobRecord[] = []
+      for (const id of ids) {
+        const job = this.readJob(id)
+        if (job) jobs.push(job)
+      }
+      return jobs
+    } catch {
+      return []
+    }
+  }
+}
+
+export const syncStateStore = new SyncStateStore()
+
+/** jobId 形如 job_20260919_0001。同日按时间戳后四位递增, 保证可排序。 */
+export function makeJobId(now = new Date()): string {
+  const stamp = [
+    now.getFullYear(),
+    String(now.getMonth() + 1).padStart(2, '0'),
+    String(now.getDate()).padStart(2, '0'),
+  ].join('')
+  return `job_${stamp}_${String(now.getTime() % 10000).padStart(4, '0')}`
+}