weflow-cli 1.6.3 → 1.7.0

package/scripts/nt_decrypt.py

@@ -510,14 +510,24 @@ def derive_database_key(path, fallback_key, fallback_salt, passphrase=''):
         return fallback_key, fallback_salt
 
 
-def connect_message_shards(db_path, key_hex, salt_hex, passphrase=''):
-    """Open the configured database plus every sibling shard.
+def connect_message_shards_detailed(db_path, key_hex, salt_hex, passphrase=''):
+    """Open every shard and report what happened to each one.
 
-    Returns the connections that actually opened. A shard that fails is
-    skipped rather than fatal: a partially readable transcript beats a
-    command that refuses to run at all.
+    Returns (pairs, failures):
+      pairs    [(shard_path, conn)] for the shards that opened
+      failures [{"name": basename, "reason": "KEY_REJECTED" | "OPEN_FAILED"}]
+
+    A shard that fails is still skipped rather than fatal - a partially
+    readable transcript beats a command that refuses to run at all - but the
+    failure stops being invisible. `shardsFailed` is the difference between
+    "read everything" and "read what happened to be reachable", which is the
+    whole point of the coverage report.
+
+    Only the basename is reported: the caller stores this in a state file, and
+    absolute paths must not end up there.
     """
-    opened = []
+    pairs = []
+    failures = []
     for shard in discover_message_shards(db_path):
         derived_key, derived_salt = derive_database_key(
             shard, key_hex, salt_hex, passphrase)
@@ -526,16 +536,60 @@ def connect_message_shards(db_path, key_hex, salt_hex, passphrase=''):
         candidates = [(derived_key, derived_salt)]
         if (derived_key, derived_salt) != (key_hex, salt_hex):
             candidates.append((key_hex, salt_hex))
+        failure_reason = 'OPEN_FAILED'
         for candidate_key, candidate_salt in candidates:
+            conn = None
             try:
                 conn, _ = connect_nt_db(shard, candidate_key, candidate_salt)
                 # PRAGMA key alone never fails; only a read surfaces a bad key.
                 conn.execute('SELECT count(*) FROM sqlite_master').fetchone()
             except Exception:
+                # Reached the database but could not decrypt it: the key is
+                # wrong, not the file. Close before retrying, or the handle
+                # leaks and on Windows the file stays locked.
+                if conn is not None:
+                    try:
+                        conn.close()
+                    except Exception:
+                        pass
+                failure_reason = 'KEY_REJECTED'
                 continue
-            opened.append(conn)
+            pairs.append((shard, conn))
             break
-    return opened
+        else:
+            failures.append({'name': os.path.basename(shard), 'reason': failure_reason})
+    return pairs, failures
+
+
+def connect_message_shards(db_path, key_hex, salt_hex, passphrase=''):
+    """Open the configured database plus every sibling shard.
+
+    Returns the connections that actually opened. A shard that fails is
+    skipped rather than fatal: a partially readable transcript beats a
+    command that refuses to run at all.
+
+    Behaviour-preserving delegate: callers that do not ask for the failure
+    detail get exactly what they got before.
+    """
+    pairs, _failures = connect_message_shards_detailed(
+        db_path, key_hex, salt_hex, passphrase)
+    return [conn for _path, conn in pairs]
+
+
+def count_talker_rows(conn, msg_table):
+    """Rows this conversation has in one shard, or None if it has no table there.
+
+    Distinguishes "this shard holds nothing for the conversation" from "this
+    shard could not be read" - without it both look like zero.
+    """
+    try:
+        cur = conn.execute(
+            "SELECT COUNT(*) FROM sqlite_master WHERE name=?", (msg_table,))
+        if cur.fetchone()[0] == 0:
+            return None
+        return conn.execute('SELECT COUNT(*) FROM "%s"' % msg_table).fetchone()[0]
+    except Exception:
+        return None
 
 
 def msg_tables(conn):
@@ -808,12 +862,17 @@ def _message_dict(row, sender_id_map, name_map, own_wxid, is_group=False):
     }
 
 
-def get_messages(conns, talker, limit=100, offset=0, name_map=None, own_wxid=None):
+def get_messages(conns, talker, limit=100, offset=0, name_map=None, own_wxid=None,
+                 shard_names=None, shard_report=None):
     """Get messages for a specific talker, merged across every shard.
 
     Args:
         name_map: optional {wxid: display_name} dict for resolving sender names
         own_wxid: account owner wxid for self-message detection
+        shard_names: optional basename per connection, parallel to `conns`
+        shard_report: optional list to append one outcome dict per shard to.
+            Purely additive: the returned messages are identical with or
+            without it, which is asserted by test/nt_decrypt_shards_test.py.
     """
     if name_map is None:
         name_map = {}
@@ -821,19 +880,36 @@ def get_messages(conns, talker, limit=100, offset=0, name_map=None, own_wxid=Non
     msg_table = f"Msg_{hashlib.md5(talker.encode()).hexdigest()}"
     is_group = '@chatroom' in talker
 
+    def record(index, opened, has_table, rows_for_talker, reason):
+        if shard_report is None:
+            return
+        name = shard_names[index] if shard_names and index < len(shard_names) else ''
+        shard_report.append({
+            'name': name,
+            'opened': opened,
+            'hasTalkerTable': has_table,
+            'rowsForTalker': rows_for_talker,
+            'reason': reason,
+        })
+
     # Each shard only needs to yield its newest window: once every shard's rows
     # are merged and re-sorted, nothing older than that can reach this page.
     window = 0 if limit <= 0 else limit + offset
     collected = []
     found = False
 
-    for conn in conns:
+    for index, conn in enumerate(conns):
         c = conn.cursor()
         try:
             c.execute("SELECT COUNT(*) FROM sqlite_master WHERE name=?", (msg_table,))
             if c.fetchone()[0] == 0:
+                # The shard is readable and simply holds nothing for this
+                # conversation - distinct from not being readable at all.
+                record(index, True, False, None, None)
                 continue
             found = True
+            rows_for_talker = c.execute(
+                'SELECT COUNT(*) FROM "%s"' % msg_table).fetchone()[0]
 
             sql = f'''
                 SELECT local_id, server_id, local_type, sort_seq, real_sender_id,
@@ -852,7 +928,11 @@ def get_messages(conns, talker, limit=100, offset=0, name_map=None, own_wxid=Non
             c.execute("SELECT rowid, user_name FROM Name2Id")
             sender_id_map = {rowid: uname for rowid, uname in c.fetchall()}
         except Exception:
+            # This shard had the conversation's table but could not be read.
+            # Previously this was indistinguishable from "no rows here".
+            record(index, True, None, None, 'READ_FAILED')
             continue
+        record(index, True, True, rows_for_talker, None)
 
         for row in rows:
             collected.append(_message_dict(row, sender_id_map, name_map, own_wxid, is_group))
@@ -1138,6 +1218,9 @@ def main():
     msg_parser.add_argument('--contact-key', default=os.environ.get('WEFLOW_CONTACT_KEY'), help='Contact DB key hex (64 chars)')
     msg_parser.add_argument('--contact-salt', default=os.environ.get('WEFLOW_CONTACT_SALT'), help='Contact DB salt hex (32 chars)')
     msg_parser.add_argument('--own-wxid', default=os.environ.get('WEFLOW_OWN_WXID'), help='Account owner wxid (for self-message detection)')
+    # Opt-in: without it the JSON is byte-identical to before this flag existed.
+    msg_parser.add_argument('--report-shards', action='store_true',
+                            help='Add a per-shard read report to the result (additive)')
 
     # contacts command
     contacts_parser = sub.add_parser('contacts', help='List contacts')
@@ -1282,12 +1365,37 @@ def main():
             conn.close()
 
     elif args.command == 'messages':
-        conns = connect_message_shards(args.db, args.key, args.salt, getattr(args, 'passphrase', '') or '')
-        if not conns:
-            print(json.dumps({"error": "无法打开消息数据库，请检查密钥"}, ensure_ascii=True))
-            return
+        passphrase = getattr(args, 'passphrase', '') or ''
         own_wxid = getattr(args, 'own_wxid', None)
-        result = get_messages(conns, args.talker, args.limit, args.offset, contact_name_map, own_wxid)
+
+        if getattr(args, 'report_shards', False):
+            pairs, failures = connect_message_shards_detailed(
+                args.db, args.key, args.salt, passphrase)
+            if not pairs:
+                print(json.dumps({"error": "无法打开消息数据库，请检查密钥"}, ensure_ascii=True))
+                return
+            conns = [conn for _path, conn in pairs]
+            names = [os.path.basename(path) for path, _conn in pairs]
+            shard_items = []
+            result = get_messages(conns, args.talker, args.limit, args.offset,
+                                  contact_name_map, own_wxid,
+                                  shard_names=names, shard_report=shard_items)
+            # Failure entries carry no opened connection, so they are not in
+            # `conns` and could not be recorded by get_messages itself.
+            result['shards'] = {
+                'scanned': len(pairs) + len(failures),
+                'opened': len(pairs),
+                'failed': len(failures),
+                'items': failures + shard_items,
+            }
+        else:
+            conns = connect_message_shards(args.db, args.key, args.salt, passphrase)
+            if not conns:
+                print(json.dumps({"error": "无法打开消息数据库，请检查密钥"}, ensure_ascii=True))
+                return
+            result = get_messages(conns, args.talker, args.limit, args.offset,
+                                  contact_name_map, own_wxid)
+
         print(json.dumps(result, ensure_ascii=True))
         for conn in conns:
             conn.close()

package/src/core/ntCore.ts

@@ -30,8 +30,28 @@ export interface NtSessionsResult extends NtResult {
   sessions?: ChatSession[]
 }
 
+/** One shard's outcome, as reported by `nt_decrypt.py --report-shards`. */
+export interface ShardOutcome {
+  /** basename only - the child never reports a full path */
+  name: string
+  opened: boolean
+  /** null when the shard could not be read at all */
+  hasTalkerTable: boolean | null
+  rowsForTalker: number | null
+  /** null | KEY_REJECTED | OPEN_FAILED | READ_FAILED */
+  reason: string | null
+}
+
+export interface ShardReport {
+  scanned: number
+  opened: number
+  failed: number
+  items: ShardOutcome[]
+}
+
 export interface NtMessagesResult extends NtResult {
   messages?: Message[]
+  shards?: ShardReport
 }
 
 export interface NtContactsResult extends NtResult {
@@ -282,6 +302,28 @@ export class NtCore {
     return { success: true, messages: result.messages || [] }
   }
 
+  /**
+   * `getMessages` plus the per-shard read report.
+   *
+   * Separate method rather than a flag on `getMessages`: the extra argv token
+   * changes what the child returns, and every existing caller is better off
+   * with the smaller payload it already parses.
+   */
+  async getMessagesWithShards(talker: string, limit = 100, offset = 0): Promise<NtMessagesResult> {
+    const args: string[] = [
+      'messages',
+      '--limit', String(limit),
+      '--offset', String(offset),
+      '--report-shards',
+    ]
+    const ownWxid = configService.get('wxid')
+    const result = await this.callPython(args, { talker, ownWxid })
+    if (result.error) {
+      return { success: false, error: result.error }
+    }
+    return { success: true, messages: result.messages || [], shards: result.shards }
+  }
+
   async getContacts(keyword?: string, limit = 200): Promise<NtContactsResult> {
     const args: string[] = [
       'contacts',

package/src/services/chatService.ts

@@ -3,6 +3,7 @@ import { existsSync } from 'fs'
 import { WcdbCore } from '../core/wcdbCore.js'
 import { SqlcipherCore } from '../core/sqlcipherCore.js'
 import { NtCore } from '../core/ntCore.js'
+import type { ShardReport } from '../core/ntCore.js'
 import { configService } from './configService.js'
 import type { ChatSession, Message, Contact, DataVersion } from '../types.js'
 import { collectMessagesInRange } from './messageQuery.js'
@@ -239,6 +240,31 @@ export class ChatService {
     return []
   }
 
+  /**
+   * Messages plus a per-shard read report, when the backend can produce one.
+   *
+   * Only the NT path (Python) knows about shards. The legacy 3.x, raw-4.x and
+   * WCDB paths return `shards: undefined`, which callers must read as "this
+   * backend cannot tell you" rather than "everything was read" - see
+   * `coverage: 'unverified'` in docs/SYNC_CONTRACT.md.
+   */
+  async getMessagesWithShards(
+    talker: string,
+    limit = 0,
+    offset = 0
+  ): Promise<{ messages: Message[]; shards?: ShardReport }> {
+    const conn = await this.connect()
+    if (!conn.success) return { messages: [] }
+
+    if (this.ntCore) {
+      const result = await this.ntCore.getMessagesWithShards(talker, limit, offset)
+      if (!result.success || !result.messages) return { messages: [] }
+      return { messages: result.messages, shards: result.shards }
+    }
+
+    return { messages: await this.getMessages(talker, limit, offset) }
+  }
+
   async getMessagesInRange(
     talker: string,
     limit = 0,

package/src/services/messageQuery.ts

@@ -36,3 +36,100 @@ export async function collectMessagesInRange(
 
   return result
 }
+
+/** Why the paging loop stopped. The first two are expected; the last two are not. */
+export type StopReason =
+  /** Reached the requested `from` bound. */
+  | 'range'
+  /** A page came back empty - there is nothing older. */
+  | 'exhausted'
+  /** Collected the caller's limit. */
+  | 'limit'
+  /** A page shorter than pageSize ended the loop before `from` was reached. */
+  | 'short-page'
+
+export interface DetailedMessageCollection {
+  messages: Message[]
+  pages: number
+  stoppedBy: StopReason
+  mayHaveMore: boolean
+  /**
+   * A short page ended the loop while the requested `from` was still ahead
+   * of us.
+   *
+   * `collectMessagesInRange` breaks on a short page, which is right when the
+   * page is short because the conversation ended - but a live database can
+   * also return a short page while rows are being written, because offset
+   * paging shifts underneath. The two are indistinguishable from the result
+   * alone, so this records the difference instead of assuming the benign one.
+   */
+  truncatedEarly: boolean
+}
+
+/**
+ * `collectMessagesInRange` plus the reason it stopped.
+ *
+ * Kept separate rather than folded into the original: that one is covered by
+ * existing tests and used by every export path, and changing what it returns
+ * would change those callers.
+ */
+export async function collectMessagesInRangeDetailed(
+  fetchPage: (limit: number, offset: number) => Promise<Message[]>,
+  limit: number,
+  range: MessageTimeRange,
+  pageSize = 500,
+): Promise<DetailedMessageCollection> {
+  const messages: Message[] = []
+  let offset = 0
+  let pages = 0
+  let stoppedBy: StopReason = 'exhausted'
+  let reachedFrom = range.from === undefined
+
+  while (true) {
+    const page = await fetchPage(pageSize, offset)
+    pages += 1
+    if (page.length === 0) {
+      stoppedBy = 'exhausted'
+      reachedFrom = true
+      break
+    }
+
+    let oldestTimestamp = Number.POSITIVE_INFINITY
+    for (const message of page) {
+      const timestamp = Number(message.createTime)
+      if (!Number.isFinite(timestamp)) continue
+      oldestTimestamp = Math.min(oldestTimestamp, timestamp)
+      if (range.from !== undefined && timestamp < range.from) continue
+      if (range.to !== undefined && timestamp > range.to) continue
+      messages.push(message)
+      if (limit > 0 && messages.length >= limit) {
+        return { messages, pages, stoppedBy: 'limit', mayHaveMore: true, truncatedEarly: false }
+      }
+    }
+
+    offset += page.length
+    if (range.from !== undefined && oldestTimestamp < range.from) {
+      stoppedBy = 'range'
+      reachedFrom = true
+      break
+    }
+    if (page.length < pageSize) {
+      // A short *first* page is unambiguous: nothing has shifted yet, so the
+      // conversation simply has fewer than pageSize messages. A short later
+      // page is the ambiguous one offset paging can produce on a live
+      // database, so it keeps its own name.
+      stoppedBy = pages === 1 ? 'exhausted' : 'short-page'
+      break
+    }
+  }
+
+  return {
+    messages,
+    pages,
+    stoppedBy,
+    // A limit hit is handled above; anything else that is not 'range' may
+    // still have older matching rows we never looked at.
+    mayHaveMore: stoppedBy !== 'range' && stoppedBy !== 'exhausted',
+    truncatedEarly: stoppedBy === 'short-page' && !reachedFrom,
+  }
+}