whalibmob 5.25.0 → 5.26.1

package/index.d.ts

@@ -665,7 +665,7 @@ export interface WhalibmobEvents {
   privacy_settings: (p: { changes: any; settings: PrivacySettings }) => void;
 
   app_state_sync: (r: AppStateSyncResult) => void;
-  app_state_mutation: (m: { collection: string; index: any; action: any; removed: boolean }) => void;
+  app_state_mutation: (m: { collection: string; index: any; action: any; removed: boolean; version?: number }) => void;
   app_state_key_missing: (m: { collection: string; keyId: string }) => void;
   app_state_keys: (m: { keys: any[] }) => void;
 
@@ -821,6 +821,15 @@ export declare class WhalibmobClient extends EventEmitter {
   syncAppState(names?: string[] | null, opts?: { snapshot?: boolean }): Promise<AppStateSyncResult>;
   /** Whether an app-state key is available — `false` on an SMS session with no companion. */
   canSyncAppState(): boolean;
+  /**
+   * Ask the primary device for app-state keys it never shared.
+   *
+   * Sent automatically when a sync runs into a key it does not have, so calling
+   * it is rarely necessary. Ids may be base64 or hex. The same id is not asked
+   * for twice within 24 hours; resolves to `null` when everything passed was
+   * held down, or when there is no connection to send it over.
+   */
+  requestAppStateKeys(keyIds: string[]): Promise<string | null>;
 
   // ─── Account restriction ─────────────────────────────────────────────────
   fetchReachoutTimelock(): Promise<ReachoutTimelockState>;

package/lib/Client.js

@@ -79,6 +79,17 @@ const MEX_QUERY_REACHOUT_TIMELOCK = '23983697327930364';
 // server takes a longer one, answers <iq type="result"/> and keeps the old text.
 const ABOUT_MAX_LENGTH = 139;
 
+// How many times a single syncAppState call will go back for more. A busy
+// account really does need several rounds, so this is high enough not to cut a
+// genuine sync short; it exists only so a server that never stops saying
+// has_more_patches cannot keep us here indefinitely.
+const MAX_APP_STATE_ROUNDS = 30;
+
+// How long to wait before asking the primary device for the same app-state key
+// again. A key it has not shared is usually one it will not share; asking on
+// every reconnect would be a message to your own phone every few seconds.
+const APP_STATE_KEY_REQUEST_INTERVAL_MS = 24 * 60 * 60 * 1000;
+
 // How long the first message to a contact waits for its trusted-contact token.
 // One round-trip to the server, no more: past this the message goes out without
 // a token rather than sitting in the queue, and the issuance finishes in the
@@ -1999,11 +2010,13 @@ class WhalibmobClient extends EventEmitter {
     const nodes = wanted.map(name => {
       const st = this._appState.get(name);
       const wantSnapshot = !!opts.snapshot || !st.version;
-      return new BinaryNode('collection', {
-        name,
-        version:         String(wantSnapshot ? 0 : st.version),
-        return_snapshot: wantSnapshot ? 'true' : 'false'
-      }, null);
+      // Asking for a snapshot means asking for the collection from nothing, so
+      // there is no version to ask from. The attribute is left off entirely
+      // rather than sent as 0 — that is what the official client puts on the
+      // wire, and a version on a snapshot request is a contradiction.
+      const attrs = { name, return_snapshot: wantSnapshot ? 'true' : 'false' };
+      if (!wantSnapshot) attrs.version = String(st.version);
+      return new BinaryNode('collection', attrs, null);
     });
 
     const resp = await this._sendIq(new BinaryNode('iq', {
@@ -2043,6 +2056,79 @@ class WhalibmobClient extends EventEmitter {
     return out;
   }
 
+  /**
+   * Ask the primary device for app-state keys it never shared.
+   *
+   * App-state mutations are encrypted under keys the phone hands out in an
+   * encrypted message of its own. Miss that message — linked while the phone
+   * was off, a session that had to be rebuilt — and every patch signed with
+   * that key is unreadable, and stays unreadable: the key is not re-offered,
+   * and nothing later in the collection can be decoded past it. The request
+   * goes out as a peer message, the same channel a placeholder resend uses, so
+   * the server routes it to your own devices rather than delivering it as a
+   * chat message to yourself.
+   *
+   * Asking twice for the same key inside a day achieves nothing, so each id is
+   * remembered and held down for that long. This is what whatsmeow does with
+   * `requestAppStateKeys`, and for the same reason.
+   *
+   * @param {string[]} keyIds  base64 (or hex) key ids, as the decoder reports
+   * @returns {Promise<string|null>} the id of the message that carried it
+   */
+  async requestAppStateKeys(keyIds) {
+    if (!Array.isArray(keyIds) || !keyIds.length) return null;
+    if (!this._sender || !this._connected) return null;
+    const me = this._store && this._store.me && this._store.me.id;
+    if (!me) return null;
+
+    this._appStateKeyRequests = this._appStateKeyRequests || new Map();
+    const now  = Date.now();
+    const want = [];
+
+    for (const id of keyIds) {
+      if (!id) continue;
+      // The decoder reports base64; a caller with the stored spelling has hex.
+      // Both name the same bytes, so normalise before deduplicating.
+      let raw;
+      try {
+        raw = /^[0-9a-fA-F]+$/.test(id) && id.length % 2 === 0
+          ? Buffer.from(id, 'hex')
+          : Buffer.from(id, 'base64');
+      } catch (_) { continue; }
+      if (!raw || !raw.length) continue;
+
+      const key  = raw.toString('base64');
+      const last = this._appStateKeyRequests.get(key);
+      if (last && now - last < APP_STATE_KEY_REQUEST_INTERVAL_MS) continue;
+      this._appStateKeyRequests.set(key, now);
+      want.push(raw);
+    }
+    if (!want.length) return null;
+
+    const {
+      encodeMessage, encodeProtocolMessage, encodeAppStateSyncKeyRequest,
+      PROTOCOL_TYPE_APP_STATE_SYNC_KEY_REQUEST
+    } = require('./proto/MessageProto');
+
+    const payload = encodeProtocolMessage({
+      type: PROTOCOL_TYPE_APP_STATE_SYNC_KEY_REQUEST,
+      appStateSyncKeyRequest: encodeAppStateSyncKeyRequest(want)
+    });
+    const msgBuf = encodeMessage('protocol', payload);
+    const msgId  = this._genMsgId();
+
+    _whaDbg('[DBG] APP_STATE_KEY_REQUEST ' +
+      want.map(b => b.toString('hex')).join(', '));
+
+    await this._sender._sendMessage(String(me), msgId, msgBuf, 'text', {
+      _protocol:      true,           // not a reach-out; keeps it off the tcToken gate
+      category:       'peer',
+      pushPriority:   'high_force',
+      additionalNodes: [new BinaryNode('meta', { appdata: 'default' }, null)]
+    });
+    return msgId;
+  }
+
   async _applyAppStateResponse(resp, opts) {
     const syncNode = findChild(resp, 'sync');
     const cols     = (syncNode && Array.isArray(syncNode.content))
@@ -2089,11 +2175,13 @@ class WhalibmobClient extends EventEmitter {
         }
       } catch (err) {
         if (err && err.isMissingKey) {
-          // Nothing can be read until the primary shares that key. Leave the
+          // Nothing in this collection can be read until the primary shares
+          // that key, and it will not offer one unasked — so ask, and leave the
           // collection where it is so the next sync picks up from here.
           _whaDbg('[DBG] APP_STATE ' + name + ' waiting for key ' + err.keyId);
           summary[name] = { waitingForKey: err.keyId };
           this.emit('app_state_key_missing', { collection: name, keyId: err.keyId });
+          this.requestAppStateKeys([err.keyId]).catch(() => {});
           continue;
         }
         throw err;
@@ -2123,10 +2211,23 @@ class WhalibmobClient extends EventEmitter {
       if (col.attrs && col.attrs.has_more_patches === 'true') more.push(name);
     }
 
-    if (more.length && !opts._retried) {
-      const again = await this.syncAppState(more, Object.assign({}, opts, { _retried: true }));
-      applied += again.applied;
-      Object.assign(summary, again.collections);
+    // The server hands a long collection over in pieces and says so with
+    // has_more_patches; a hash that stopped agreeing puts the collection back
+    // here too, once, as a snapshot. Either way there is more to fetch, and
+    // stopping after one more round would leave the collection half-read
+    // without anything saying so. Keep going until the server stops asking —
+    // with a ceiling, because a server that always says "more" would otherwise
+    // spin here forever.
+    if (more.length) {
+      const rounds = (opts._round || 0) + 1;
+      if (rounds > MAX_APP_STATE_ROUNDS) {
+        _whaDbg('[DBG] APP_STATE giving up after ' + rounds + ' rounds: ' + more.join(', '));
+      } else {
+        const again = await this.syncAppState(more,
+          Object.assign({}, opts, { _round: rounds, snapshot: false }));
+        applied += again.applied;
+        Object.assign(summary, again.collections);
+      }
     }
 
     this.emit('app_state_sync', { collections: summary, applied });
@@ -2254,7 +2355,7 @@ class WhalibmobClient extends EventEmitter {
           // Something the server tracks that this library does not model yet.
           // Reported rather than dropped, so it is visible that it happened.
           this.emit('app_state_mutation', {
-            collection, index: m.index, action: a, removed
+            collection, index: m.index, action: a, removed, version: m.version
           });
           break;
       }
@@ -5360,9 +5461,9 @@ class WhalibmobClient extends EventEmitter {
     // Web-mode history lands in its own files. A number can be registered over
     // SMS and separately linked as a companion, and only the companion ever
     // receives history — merging the two would be merging different accounts'
-    // views of the same number.
-    const phone = (this._store && this._store.phoneNumber) +
-                  (this._mode === 'web' ? '.web' : '');
+    // views of the same number. Same prefix the readers use.
+    const phone = this._sessionFilePrefix();
+    if (!phone) return;
     const sessionDir = this._sessionDir;
 
     _whaDbg('[DBG] HIST_SYNC start syncType=' + notification.syncType +
@@ -5510,6 +5611,19 @@ class WhalibmobClient extends EventEmitter {
       _whaDbg('[DBG] APP_STATE_KEY_SHARE_SAVE_ERR ' + e.message);
     }
 
+    // A key that has arrived is no longer one we are holding a request down
+    // for; drop the hold so that if it is ever missing again the ask goes out
+    // at once rather than a day later.
+    if (this._appStateKeyRequests) {
+      for (const k of keyShare.keys) {
+        if (!k || !k.keyId) continue;
+        try {
+          this._appStateKeyRequests.delete(
+            Buffer.from(k.keyId, 'hex').toString('base64'));
+        } catch (_) {}
+      }
+    }
+
     this.emit('app_state_keys', { keys: keyShare.keys });
 
     // The key is what app state was waiting on. Anything the server has been
@@ -5566,15 +5680,25 @@ class WhalibmobClient extends EventEmitter {
   }
 
   /**
-   * getHistoryStore — load the persisted history store for this phone.
-   * Returns the parsed {phone}.history.json content or null if not yet synced.
+   * The name every file of this session is built from.
    *
-   * @returns {object|null}
+   * A number can be registered over SMS and separately linked as a companion,
+   * and the two are different accounts' views of it, so the companion's files
+   * carry a `.web` in the middle: 40712345678.web.history.json. Everything that
+   * reads or writes one has to agree on that — a writer that adds the `.web`
+   * and a reader that does not will not error, it will simply never find the
+   * file it just wrote.
    */
-  getHistoryStore() {
+  _sessionFilePrefix() {
     if (!this._store || !this._store.phoneNumber) return null;
+    return this._store.phoneNumber + (this._mode === 'web' ? '.web' : '');
+  }
+
+  _readSessionJson(suffix) {
+    const prefix = this._sessionFilePrefix();
+    if (!prefix) return null;
     const fs   = require('fs');
-    const file = require('path').join(this._sessionDir, this._store.phoneNumber + '.history.json');
+    const file = require('path').join(this._sessionDir, prefix + suffix);
     try {
       if (!fs.existsSync(file)) return null;
       return JSON.parse(fs.readFileSync(file, 'utf8'));
@@ -5582,19 +5706,25 @@ class WhalibmobClient extends EventEmitter {
   }
 
   /**
-   * getMessagesStore — load the persisted messages for this phone.
-   * Returns the parsed {phone}.messages.json content or null if not yet synced.
+   * getHistoryStore — load the persisted history store for this session.
+   * Returns the parsed {phone}.history.json content — {phone}.web.history.json
+   * on a companion — or null if not yet synced.
+   *
+   * @returns {object|null}
+   */
+  getHistoryStore() {
+    return this._readSessionJson('.history.json');
+  }
+
+  /**
+   * getMessagesStore — load the persisted messages for this session.
+   * Returns the parsed {phone}.messages.json content — {phone}.web.messages.json
+   * on a companion — or null if not yet synced.
    *
    * @returns {object|null}
    */
   getMessagesStore() {
-    if (!this._store || !this._store.phoneNumber) return null;
-    const fs   = require('fs');
-    const file = require('path').join(this._sessionDir, this._store.phoneNumber + '.messages.json');
-    try {
-      if (!fs.existsSync(file)) return null;
-      return JSON.parse(fs.readFileSync(file, 'utf8'));
-    } catch (_) { return null; }
+    return this._readSessionJson('.messages.json');
   }
 
   // ─── Public API ───────────────────────────────────────────────────────────

package/lib/appstate/AppStateSync.js

@@ -95,8 +95,8 @@ class MissingAppStateKeyError extends Error {
  * missing key is different — nothing after it can be read either, and the fix
  * is to wait for the key rather than to carry on — so that one is thrown.
  */
-function decodeMutations(mutations, state, getKey, onMutation, validateMacs) {
-  const gen   = makeGenerator(state);
+function decodeMutations(mutations, state, getKey, onMutation, validateMacs, opts) {
+  const gen   = makeGenerator(state, opts);
   const cache = new Map();
   let skipped = 0;
 
@@ -145,7 +145,17 @@ function decodeMutations(mutations, state, getKey, onMutation, validateMacs) {
     try { index = JSON.parse(data.index.toString('utf8')); }
     catch (_) { skipped++; continue; }
 
-    onMutation({ index, action: data.value, operation: m.operation });
+    onMutation({
+      index,
+      action:    data.value,
+      operation: m.operation,
+      // The api version the mutation was written under. The server sends it, so
+      // report it: a caller that has to tell an old write from a new one has
+      // nothing else to go on.
+      version:   data.version,
+      indexMac:  m.indexBlob,
+      valueMac:  theirMac
+    });
     gen.mix({
       indexMac: m.indexBlob,
       valueMac: theirMac,
@@ -168,7 +178,8 @@ function decodeSnapshot(name, snapshot, getKey, getMutation, validateMacs) {
   const state = newState();
   state.version = snapshot.version;
 
-  const res = decodeMutations(snapshot.records, state, getKey, getMutation, validateMacs);
+  const res = decodeMutations(snapshot.records, state, getKey, getMutation,
+    validateMacs, { snapshot: true });
   state.hash          = res.hash;
   state.indexValueMap = res.indexValueMap;
 

package/lib/appstate/LTHash.js

@@ -79,7 +79,15 @@ const PATCH_INTEGRITY = new LTHash(PATCH_SALT);
 // indexValueMap is index MAC (base64) → { valueMac }. It is the reason a
 // mutation can be replaced at all: without knowing the value an index used to
 // carry, there is nothing to subtract back out.
-function makeGenerator(state) {
+//
+// `snapshot` turns the subtraction off. A snapshot is the whole collection
+// stated from nothing, so the server computed its hash by adding every record
+// and subtracting none — including two records that happen to share an index.
+// Displacing the first would leave us one value short of the hash the snapshot
+// is signed against, and the collection would be thrown away for a mismatch
+// that was ours.
+function makeGenerator(state, opts) {
+  const snapshot      = !!(opts && opts.snapshot);
   const indexValueMap = Object.assign({}, state.indexValueMap || {});
   const hash    = Buffer.isBuffer(state.hash) ? state.hash : Buffer.alloc(HASH_BYTES);
   const addBufs = [];
@@ -88,7 +96,7 @@ function makeGenerator(state) {
   return {
     mix({ indexMac, valueMac, remove }) {
       const key    = Buffer.from(indexMac).toString('base64');
-      const prev   = indexValueMap[key];
+      const prev   = snapshot ? null : indexValueMap[key];
       if (remove) {
         // A remove for something we never had is the server telling us about a
         // mutation we missed. There is nothing to subtract; the hash will not

package/lib/proto/MessageProto.js

@@ -364,18 +364,25 @@ function encodeDeviceSentMessage(destinationJid, messageBuf, phash) {
 }
 
 // ─── ProtocolMessage (field 12 of Message) ───────────────────────────────────
-// ProtocolMessage { key=1, type=2, ephemeralExpiration=4, ... }
-// Type enum: REVOKE=0, EPHEMERAL_SETTING=3
+// ProtocolMessage { key=1, type=2, ephemeralExpiration=4,
+//                   appStateSyncKeyShare=6, appStateSyncKeyRequest=8, ... }
+// Type enum: REVOKE=0, EPHEMERAL_SETTING=3, APP_STATE_SYNC_KEY_REQUEST=7
 const PROTOCOL_MSG_REVOKE    = 0;
 const PROTOCOL_MSG_EPHEMERAL = 3;
 
+/** ProtocolMessage.Type.APP_STATE_SYNC_KEY_REQUEST */
+const PROTOCOL_TYPE_APP_STATE_SYNC_KEY_REQUEST = 7;
+
 function encodeProtocolMessage(opts) {
   // opts: { type, key: { remoteJid, fromMe, id }, ephemeralExpiration,
-  //         peerDataOperationRequestMessage }
+  //         appStateSyncKeyRequest, peerDataOperationRequestMessage }
   const parts = [];
   if (opts.key) parts.push(field(1, WIRE_LEN, encodeMessageKey(opts.key)));
   if (opts.type !== undefined) parts.push(field(2, WIRE_VARINT, varint(opts.type)));
   if (opts.ephemeralExpiration !== undefined) parts.push(field(4, WIRE_VARINT, varint(opts.ephemeralExpiration)));
+  if (opts.appStateSyncKeyRequest) {
+    parts.push(field(8, WIRE_LEN, opts.appStateSyncKeyRequest));
+  }
   // Field 16, and the Type enum value that goes with it is also 16 — a
   // coincidence in the schema, not a mistake here.
   if (opts.peerDataOperationRequestMessage) {
@@ -384,6 +391,21 @@ function encodeProtocolMessage(opts) {
   return Buffer.concat(parts);
 }
 
+// ─── AppStateSyncKeyRequest ──────────────────────────────────────────────────
+//
+// Asking the primary device for an app-state key it never shared. Mutations are
+// encrypted under a key the phone hands out over an encrypted message; if that
+// message was missed, every patch signed with that key is unreadable and stays
+// unreadable — nothing later in the collection can be decoded either. The only
+// way out is to ask, which is what this carries.
+//
+//   AppStateSyncKeyRequest { keyIds = 1 (repeated) }
+//   AppStateSyncKeyId      { keyId  = 1 }
+function encodeAppStateSyncKeyRequest(keyIds) {
+  return Buffer.concat((keyIds || []).map(
+    id => field(1, WIRE_LEN, field(1, WIRE_LEN, bytes(id)))));
+}
+
 // ─── PeerDataOperationRequestMessage ─────────────────────────────────────────
 //
 // The channel a companion asks its own phone for something over. It travels as
@@ -980,6 +1002,8 @@ module.exports = {
   encodePeerDataOperationRequestMessage,
   PeerDataOperationRequestType,
   PROTOCOL_TYPE_PEER_DATA_OPERATION_REQUEST,
+  PROTOCOL_TYPE_APP_STATE_SYNC_KEY_REQUEST,
+  encodeAppStateSyncKeyRequest,
   encodeExtendedText,
   encodeContextInfo,
   encodeSenderKeyDistributionMessage,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "whalibmob",
-  "version": "5.25.0",
+  "version": "5.26.1",
   "description": "Node.js library for WhatsApp — register a number over SMS, or link as a companion by QR. Signal E2E encryption, media, groups, channels.",
   "author": "Kunboruto20",
   "main": "index.js",