weflow-cli 1.6.4 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/OPERATIONS.md +35 -0
- package/bin/weflow-cli.ts +312 -0
- package/dist/bin/weflow-cli.js +352 -0
- package/dist/bin/weflow-cli.js.map +1 -1
- package/dist/src/core/ntCore.d.ts +26 -0
- package/dist/src/core/ntCore.d.ts.map +1 -1
- package/dist/src/core/ntCore.js +21 -0
- package/dist/src/core/ntCore.js.map +1 -1
- package/dist/src/services/chatService.d.ts +13 -0
- package/dist/src/services/chatService.d.ts.map +1 -1
- package/dist/src/services/chatService.js +20 -0
- package/dist/src/services/chatService.js.map +1 -1
- package/dist/src/services/messageQuery.d.ts +35 -0
- package/dist/src/services/messageQuery.d.ts.map +1 -1
- package/dist/src/services/messageQuery.js +61 -0
- package/dist/src/services/messageQuery.js.map +1 -1
- package/dist/src/services/syncService.d.ts +121 -0
- package/dist/src/services/syncService.d.ts.map +1 -0
- package/dist/src/services/syncService.js +244 -0
- package/dist/src/services/syncService.js.map +1 -0
- package/dist/src/services/syncState.d.ts +113 -0
- package/dist/src/services/syncState.d.ts.map +1 -0
- package/dist/src/services/syncState.js +214 -0
- package/dist/src/services/syncState.js.map +1 -0
- package/docs/DECISIONS.md +60 -20
- package/docs/FUNCTIONAL-IMPROVEMENT-SPEC.md +338 -0
- package/docs/PROJECT_STATE.md +5 -3
- package/docs/SYNC_CONTRACT.md +152 -0
- package/package.json +1 -1
- package/scripts/export_chat_html.py +177 -13
- package/scripts/nt_decrypt.py +123 -15
- package/src/core/ntCore.ts +42 -0
- package/src/services/chatService.ts +26 -0
- package/src/services/messageQuery.ts +97 -0
- package/src/services/syncService.ts +335 -0
- package/src/services/syncState.ts +275 -0
|
@@ -124,6 +124,12 @@ VOICE_STATE = {'map': {}, 'cache': None}
|
|
|
124
124
|
# runs a throwaway formatting pass that only records the URLs (0.1s, no
|
|
125
125
|
# network), fetches them concurrently, then formats for real against a warm
|
|
126
126
|
# cache. Set to a list during that pass; None otherwise.
|
|
127
|
+
# Why the most recent remote fetch produced nothing. The skip counter alone
|
|
128
|
+
# said "not fetched" without saying why, so "no budget left" and "the URL is
|
|
129
|
+
# dead" were indistinguishable in the export - and both looked like a bare
|
|
130
|
+
# placeholder to the reader.
|
|
131
|
+
MEDIA_STATE = {'remote_reason': None}
|
|
132
|
+
|
|
127
133
|
PREFETCH = {'sink': None, 'active': False}
|
|
128
134
|
PREFETCH_WORKERS = 24
|
|
129
135
|
PREFETCH_MAX_URLS = 800
|
|
@@ -752,13 +758,13 @@ def shrink_embedded(image, max_side=720, force=False):
|
|
|
752
758
|
return image
|
|
753
759
|
|
|
754
760
|
|
|
755
|
-
def
|
|
756
|
-
"""Best local media for a message
|
|
761
|
+
def resolve_media(image_map, local_id, create_time, content='', resource_md5s=None, server_id=0):
|
|
762
|
+
"""Best local media for a message, with the identity that matched.
|
|
757
763
|
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
764
|
+
Returns `(base64, mime, media_key)` or None. `media_key` is the image_map
|
|
765
|
+
key that produced the hit, which is what the coverage report records - so
|
|
766
|
+
a report can never claim a match the page itself did not get, because both
|
|
767
|
+
come from this one function.
|
|
762
768
|
|
|
763
769
|
Deliberately *not* falling back to a bare `local_id`: that id restarts in
|
|
764
770
|
every message shard, so matching on it alone attaches one conversation's
|
|
@@ -769,26 +775,40 @@ def get_cached_image(image_map, local_id, create_time, content='', resource_md5s
|
|
|
769
775
|
if not image_map:
|
|
770
776
|
return None
|
|
771
777
|
for media_md5 in resource_md5s or []:
|
|
772
|
-
|
|
773
|
-
|
|
778
|
+
key = f'md5:{media_md5}'
|
|
779
|
+
if image_map.get(key):
|
|
780
|
+
return shrink_embedded(image_map[key]) + (key,)
|
|
774
781
|
for media_md5 in extract_media_md5s(content):
|
|
775
|
-
|
|
776
|
-
|
|
782
|
+
key = f'md5:{media_md5}'
|
|
783
|
+
if image_map.get(key):
|
|
784
|
+
return shrink_embedded(image_map[key]) + (key,)
|
|
777
785
|
try:
|
|
778
786
|
pair_key = f'pair:{int(local_id)}:{int(create_time)}'
|
|
779
787
|
except (TypeError, ValueError):
|
|
780
788
|
return None
|
|
781
789
|
if image_map.get(pair_key):
|
|
782
|
-
return shrink_embedded(image_map[pair_key])
|
|
790
|
+
return shrink_embedded(image_map[pair_key]) + (pair_key,)
|
|
783
791
|
try:
|
|
784
792
|
unique_key = f'unique:{int(local_id)}'
|
|
785
793
|
except (TypeError, ValueError):
|
|
786
794
|
return None
|
|
787
795
|
if image_map.get(unique_key):
|
|
788
|
-
return shrink_embedded(image_map[unique_key])
|
|
796
|
+
return shrink_embedded(image_map[unique_key]) + (unique_key,)
|
|
789
797
|
return None
|
|
790
798
|
|
|
791
799
|
|
|
800
|
+
def get_cached_image(image_map, local_id, create_time, content='', resource_md5s=None, server_id=0):
|
|
801
|
+
"""`resolve_media` without the matched key, for every pre-existing caller.
|
|
802
|
+
|
|
803
|
+
md5 is the strongest key, but it is not always derivable - a group row can
|
|
804
|
+
lack the resource mapping that carries it. The conversation cache also
|
|
805
|
+
indexes by `(local_id, create_time)`, which is identity-aware and was
|
|
806
|
+
already being stored; reading it recovers media the md5 path alone misses.
|
|
807
|
+
"""
|
|
808
|
+
hit = resolve_media(image_map, local_id, create_time, content, resource_md5s, server_id)
|
|
809
|
+
return (hit[0], hit[1]) if hit else None
|
|
810
|
+
|
|
811
|
+
|
|
792
812
|
def detect_mime(filepath):
|
|
793
813
|
"""Detect MIME type from file header."""
|
|
794
814
|
try:
|
|
@@ -918,12 +938,14 @@ def download_image_as_base64(url, aes_key='', timeout=10, budgeted=True):
|
|
|
918
938
|
already spent their own budget for this URL.
|
|
919
939
|
"""
|
|
920
940
|
if not url or not url.startswith(('http://', 'https://')):
|
|
941
|
+
MEDIA_STATE['remote_reason'] = 'not-http'
|
|
921
942
|
return None
|
|
922
943
|
|
|
923
944
|
sink = PREFETCH['sink']
|
|
924
945
|
if sink is not None:
|
|
925
946
|
# Dry pass: record the request, spend no budget, touch no network.
|
|
926
947
|
sink.append(('img', url, aes_key))
|
|
948
|
+
MEDIA_STATE['remote_reason'] = 'deferred-to-prefetch'
|
|
927
949
|
return None
|
|
928
950
|
|
|
929
951
|
cache_dir = COVER_STATE.get('dir') or ''
|
|
@@ -935,11 +957,13 @@ def download_image_as_base64(url, aes_key='', timeout=10, budgeted=True):
|
|
|
935
957
|
if cached != 'unknown':
|
|
936
958
|
if budgeted:
|
|
937
959
|
COVER_STATE['thumb_cached'] += 1
|
|
960
|
+
MEDIA_STATE['remote_reason'] = None
|
|
938
961
|
return cached
|
|
939
962
|
|
|
940
963
|
if budgeted and not PREFETCH['active']:
|
|
941
964
|
if COVER_STATE.get('thumb_budget', 0) <= 0:
|
|
942
965
|
COVER_STATE['thumb_skipped'] += 1
|
|
966
|
+
MEDIA_STATE['remote_reason'] = 'budget-exhausted'
|
|
943
967
|
return None
|
|
944
968
|
COVER_STATE['thumb_budget'] -= 1
|
|
945
969
|
COVER_STATE['thumb_fetched'] += 1
|
|
@@ -956,6 +980,7 @@ def download_image_as_base64(url, aes_key='', timeout=10, budgeted=True):
|
|
|
956
980
|
# it so the next export does not pay for the same dead URL.
|
|
957
981
|
if len(data) > MAX_EMBED_SIZE:
|
|
958
982
|
_cache_media(cache_file, None)
|
|
983
|
+
MEDIA_STATE['remote_reason'] = 'too-large'
|
|
959
984
|
return None
|
|
960
985
|
candidates = [data]
|
|
961
986
|
decrypted = _decrypt_aes_cbc(data, aes_key)
|
|
@@ -969,10 +994,12 @@ def download_image_as_base64(url, aes_key='', timeout=10, budgeted=True):
|
|
|
969
994
|
data, mime = wechat_image.shrink(data, mime, max_side=480)
|
|
970
995
|
result = (base64.b64encode(data).decode(), mime)
|
|
971
996
|
_cache_media(cache_file, result)
|
|
997
|
+
MEDIA_STATE['remote_reason'] = None
|
|
972
998
|
return result
|
|
973
999
|
except Exception:
|
|
974
1000
|
continue
|
|
975
1001
|
_cache_media(cache_file, None)
|
|
1002
|
+
MEDIA_STATE['remote_reason'] = 'download-failed'
|
|
976
1003
|
return None
|
|
977
1004
|
|
|
978
1005
|
|
|
@@ -1891,9 +1918,117 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
1891
1918
|
'local_type': local_type,
|
|
1892
1919
|
'display': display,
|
|
1893
1920
|
'image_b64': image_b64,
|
|
1921
|
+
# Built here rather than accumulated in module state: format_message
|
|
1922
|
+
# runs twice (the prefetch dry sweep and the real pass) and the dry
|
|
1923
|
+
# pass is discarded, so a counter would double-count.
|
|
1924
|
+
'media': build_media_record(
|
|
1925
|
+
local_type, image_b64, image_map, local_id, create_time, content,
|
|
1926
|
+
resource_md5s, metadata_content, bool(server_id)),
|
|
1894
1927
|
}
|
|
1895
1928
|
|
|
1896
1929
|
|
|
1930
|
+
MEDIA_KIND_BY_TYPE = {
|
|
1931
|
+
3: 'image', 34: 'voice', 43: 'video-thumb', 47: 'emoji', 49: 'link-cover',
|
|
1932
|
+
}
|
|
1933
|
+
MEDIA_KINDS = {'image', 'emoji', 'voice', 'video-thumb', 'link-cover'}
|
|
1934
|
+
|
|
1935
|
+
|
|
1936
|
+
def build_media_report(items, scope, talker, per_page):
|
|
1937
|
+
"""Coverage summary plus one entry per media item.
|
|
1938
|
+
|
|
1939
|
+
Written beside the HTML rather than into it: the page is for reading, this
|
|
1940
|
+
is for answering "what is missing and why" without opening the export.
|
|
1941
|
+
Paths, keys and message bodies are not included (D-001/D-002/D-025).
|
|
1942
|
+
"""
|
|
1943
|
+
counts = {}
|
|
1944
|
+
for item in items:
|
|
1945
|
+
counts[item['status']] = counts.get(item['status'], 0) + 1
|
|
1946
|
+
counts['total'] = len(items)
|
|
1947
|
+
return {
|
|
1948
|
+
'schema': 'weflow-media-report/v1',
|
|
1949
|
+
'source': 'weflow-cli',
|
|
1950
|
+
'generatedAt': datetime.datetime.now().astimezone().isoformat(timespec='seconds'),
|
|
1951
|
+
'scope': scope,
|
|
1952
|
+
'counts': counts,
|
|
1953
|
+
'budget': {
|
|
1954
|
+
'coversFetched': COVER_STATE.get('fetched', 0),
|
|
1955
|
+
'coversCached': COVER_STATE.get('cached', 0),
|
|
1956
|
+
'coversSkipped': COVER_STATE.get('skipped', 0),
|
|
1957
|
+
'thumbsFetched': COVER_STATE.get('thumb_fetched', 0),
|
|
1958
|
+
'thumbsCached': COVER_STATE.get('thumb_cached', 0),
|
|
1959
|
+
'thumbsSkipped': COVER_STATE.get('thumb_skipped', 0),
|
|
1960
|
+
},
|
|
1961
|
+
'items': [
|
|
1962
|
+
{
|
|
1963
|
+
'mediaKey': item['mediaKey'],
|
|
1964
|
+
'kind': item['kind'],
|
|
1965
|
+
'status': item['status'],
|
|
1966
|
+
'mime': item['mime'],
|
|
1967
|
+
'bytes': item['bytes'],
|
|
1968
|
+
'sha256': item['sha256'],
|
|
1969
|
+
'reason': item['reason'],
|
|
1970
|
+
'locator': {
|
|
1971
|
+
'part': (item['index'] // per_page + 1) if per_page else 1,
|
|
1972
|
+
'createTime': None,
|
|
1973
|
+
},
|
|
1974
|
+
}
|
|
1975
|
+
for item in items
|
|
1976
|
+
],
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
|
|
1980
|
+
def build_media_record(local_type, image_b64, image_map, local_id, create_time,
|
|
1981
|
+
content, resource_md5s, metadata_content, has_server_id):
|
|
1982
|
+
"""What happened to this message's media, or None for a non-media message.
|
|
1983
|
+
|
|
1984
|
+
Derived from the same `resolve_media` the page used, so the report cannot
|
|
1985
|
+
claim a match the page did not get (or vice versa). A media-free message
|
|
1986
|
+
returns None: this is a media manifest, not a message manifest.
|
|
1987
|
+
"""
|
|
1988
|
+
if local_type not in MEDIA_KIND_BY_TYPE:
|
|
1989
|
+
return None
|
|
1990
|
+
kind = MEDIA_KIND_BY_TYPE[local_type]
|
|
1991
|
+
record = {
|
|
1992
|
+
'mediaKey': None, 'kind': kind, 'status': None,
|
|
1993
|
+
'mime': None, 'bytes': None, 'sha256': None, 'reason': None,
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
if image_b64:
|
|
1997
|
+
hit = resolve_media(image_map, local_id, create_time, metadata_content,
|
|
1998
|
+
resource_md5s)
|
|
1999
|
+
if hit:
|
|
2000
|
+
# Matched a local key: report which one, so "why this image" is
|
|
2001
|
+
# answerable without re-running the export.
|
|
2002
|
+
record['mediaKey'] = hit[2]
|
|
2003
|
+
record['status'] = 'embedded'
|
|
2004
|
+
else:
|
|
2005
|
+
# Bytes came from somewhere other than the conversation index.
|
|
2006
|
+
if MEDIA_STATE['remote_reason'] is None:
|
|
2007
|
+
record['status'] = 'remote-fetched'
|
|
2008
|
+
record['mediaKey'] = 'remote'
|
|
2009
|
+
else:
|
|
2010
|
+
record['status'] = 'embedded'
|
|
2011
|
+
record['mediaKey'] = 'unindexed-local'
|
|
2012
|
+
raw = base64.b64decode(image_b64)
|
|
2013
|
+
record['bytes'] = len(raw)
|
|
2014
|
+
record['sha256'] = hashlib.sha256(raw).hexdigest()
|
|
2015
|
+
return record
|
|
2016
|
+
|
|
2017
|
+
record['status'] = 'missing'
|
|
2018
|
+
if not has_server_id and not extract_media_md5s(metadata_content) and not resource_md5s:
|
|
2019
|
+
# D-014: with no reliable identity there is nothing safe to match on,
|
|
2020
|
+
# and guessing would attach someone else's media.
|
|
2021
|
+
record['reason'] = 'no-reliable-identity'
|
|
2022
|
+
elif kind == 'voice':
|
|
2023
|
+
record['status'] = 'unsupported'
|
|
2024
|
+
record['reason'] = 'voice-not-in-media-index'
|
|
2025
|
+
elif MEDIA_STATE['remote_reason']:
|
|
2026
|
+
record['reason'] = MEDIA_STATE['remote_reason']
|
|
2027
|
+
else:
|
|
2028
|
+
record['reason'] = 'not-in-local-cache'
|
|
2029
|
+
return record
|
|
2030
|
+
|
|
2031
|
+
|
|
1897
2032
|
def decode_message_content(value):
|
|
1898
2033
|
"""Decode NT message content, which may be Zstandard compressed."""
|
|
1899
2034
|
if not value:
|
|
@@ -2337,6 +2472,7 @@ def main():
|
|
|
2337
2472
|
|
|
2338
2473
|
print(f"Formatting {len(messages)} messages...")
|
|
2339
2474
|
formatted = []
|
|
2475
|
+
media_items = []
|
|
2340
2476
|
img_hit_count = 0
|
|
2341
2477
|
article_img_count = 0
|
|
2342
2478
|
progress_step = max(1, len(messages) // 20)
|
|
@@ -2348,6 +2484,12 @@ def main():
|
|
|
2348
2484
|
img_hit_count += 1
|
|
2349
2485
|
if result.get('local_type') == 49:
|
|
2350
2486
|
article_img_count += 1
|
|
2487
|
+
record = result.get('media')
|
|
2488
|
+
if record:
|
|
2489
|
+
# Index kept so the report can point back at the page part.
|
|
2490
|
+
record = dict(record)
|
|
2491
|
+
record['index'] = i
|
|
2492
|
+
media_items.append(record)
|
|
2351
2493
|
formatted.append(result)
|
|
2352
2494
|
print(f" Messages with embedded images: {img_hit_count} (including {article_img_count} article thumbnails)")
|
|
2353
2495
|
|
|
@@ -2367,6 +2509,20 @@ def main():
|
|
|
2367
2509
|
# Use display name for filename if provided, otherwise fallback to wxid
|
|
2368
2510
|
file_prefix = sanitize_filename(display_name) if display_name else args.talker.replace('@', '_').replace('/', '_')
|
|
2369
2511
|
|
|
2512
|
+
# Media coverage report. Written beside the HTML parts, under the same
|
|
2513
|
+
# prefix, so it is always found next to what it describes.
|
|
2514
|
+
media_report = None
|
|
2515
|
+
media_report_path = None
|
|
2516
|
+
if media_items:
|
|
2517
|
+
media_report = build_media_report(media_items, display_name, args.talker, per_part)
|
|
2518
|
+
media_report_path = os.path.join(args.out, f'{file_prefix}_media.json')
|
|
2519
|
+
with open(media_report_path, 'w', encoding='utf-8') as handle:
|
|
2520
|
+
json.dump(media_report, handle, ensure_ascii=False, indent=2)
|
|
2521
|
+
summary = media_report['counts']
|
|
2522
|
+
missing = summary.get('missing', 0) + summary.get('unsupported', 0)
|
|
2523
|
+
print(f"Media report: {len(media_items)} item(s), {missing} unavailable "
|
|
2524
|
+
f"-> {os.path.basename(media_report_path)}")
|
|
2525
|
+
|
|
2370
2526
|
html_files = []
|
|
2371
2527
|
for i in range(parts):
|
|
2372
2528
|
start = i * per_part
|
|
@@ -2395,12 +2551,20 @@ def main():
|
|
|
2395
2551
|
print(f"\nDone! {len(html_files)} HTML files written to {args.out}")
|
|
2396
2552
|
print(f"Total: {total} messages")
|
|
2397
2553
|
|
|
2398
|
-
# Print JSON summary for CLI integration
|
|
2554
|
+
# Print JSON summary for CLI integration.
|
|
2555
|
+
# The four original keys are untouched; `media` is purely additive and the
|
|
2556
|
+
# one existing consumer (src/services/exportService.ts) ignores it.
|
|
2399
2557
|
print(json.dumps({
|
|
2400
2558
|
"success": True,
|
|
2401
2559
|
"total": total,
|
|
2402
2560
|
"parts": len(html_files),
|
|
2403
2561
|
"files": html_files,
|
|
2562
|
+
"media": ({
|
|
2563
|
+
"schema": "weflow-media-report/v1",
|
|
2564
|
+
"records": len(media_items),
|
|
2565
|
+
"status": media_report['counts'],
|
|
2566
|
+
"report": os.path.basename(media_report_path) if media_report_path else None,
|
|
2567
|
+
} if media_report else None),
|
|
2404
2568
|
}))
|
|
2405
2569
|
|
|
2406
2570
|
|
package/scripts/nt_decrypt.py
CHANGED
|
@@ -510,14 +510,24 @@ def derive_database_key(path, fallback_key, fallback_salt, passphrase=''):
|
|
|
510
510
|
return fallback_key, fallback_salt
|
|
511
511
|
|
|
512
512
|
|
|
513
|
-
def
|
|
514
|
-
"""Open
|
|
513
|
+
def connect_message_shards_detailed(db_path, key_hex, salt_hex, passphrase=''):
|
|
514
|
+
"""Open every shard and report what happened to each one.
|
|
515
515
|
|
|
516
|
-
Returns
|
|
517
|
-
|
|
518
|
-
|
|
516
|
+
Returns (pairs, failures):
|
|
517
|
+
pairs [(shard_path, conn)] for the shards that opened
|
|
518
|
+
failures [{"name": basename, "reason": "KEY_REJECTED" | "OPEN_FAILED"}]
|
|
519
|
+
|
|
520
|
+
A shard that fails is still skipped rather than fatal - a partially
|
|
521
|
+
readable transcript beats a command that refuses to run at all - but the
|
|
522
|
+
failure stops being invisible. `shardsFailed` is the difference between
|
|
523
|
+
"read everything" and "read what happened to be reachable", which is the
|
|
524
|
+
whole point of the coverage report.
|
|
525
|
+
|
|
526
|
+
Only the basename is reported: the caller stores this in a state file, and
|
|
527
|
+
absolute paths must not end up there.
|
|
519
528
|
"""
|
|
520
|
-
|
|
529
|
+
pairs = []
|
|
530
|
+
failures = []
|
|
521
531
|
for shard in discover_message_shards(db_path):
|
|
522
532
|
derived_key, derived_salt = derive_database_key(
|
|
523
533
|
shard, key_hex, salt_hex, passphrase)
|
|
@@ -526,16 +536,60 @@ def connect_message_shards(db_path, key_hex, salt_hex, passphrase=''):
|
|
|
526
536
|
candidates = [(derived_key, derived_salt)]
|
|
527
537
|
if (derived_key, derived_salt) != (key_hex, salt_hex):
|
|
528
538
|
candidates.append((key_hex, salt_hex))
|
|
539
|
+
failure_reason = 'OPEN_FAILED'
|
|
529
540
|
for candidate_key, candidate_salt in candidates:
|
|
541
|
+
conn = None
|
|
530
542
|
try:
|
|
531
543
|
conn, _ = connect_nt_db(shard, candidate_key, candidate_salt)
|
|
532
544
|
# PRAGMA key alone never fails; only a read surfaces a bad key.
|
|
533
545
|
conn.execute('SELECT count(*) FROM sqlite_master').fetchone()
|
|
534
546
|
except Exception:
|
|
547
|
+
# Reached the database but could not decrypt it: the key is
|
|
548
|
+
# wrong, not the file. Close before retrying, or the handle
|
|
549
|
+
# leaks and on Windows the file stays locked.
|
|
550
|
+
if conn is not None:
|
|
551
|
+
try:
|
|
552
|
+
conn.close()
|
|
553
|
+
except Exception:
|
|
554
|
+
pass
|
|
555
|
+
failure_reason = 'KEY_REJECTED'
|
|
535
556
|
continue
|
|
536
|
-
|
|
557
|
+
pairs.append((shard, conn))
|
|
537
558
|
break
|
|
538
|
-
|
|
559
|
+
else:
|
|
560
|
+
failures.append({'name': os.path.basename(shard), 'reason': failure_reason})
|
|
561
|
+
return pairs, failures
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def connect_message_shards(db_path, key_hex, salt_hex, passphrase=''):
|
|
565
|
+
"""Open the configured database plus every sibling shard.
|
|
566
|
+
|
|
567
|
+
Returns the connections that actually opened. A shard that fails is
|
|
568
|
+
skipped rather than fatal: a partially readable transcript beats a
|
|
569
|
+
command that refuses to run at all.
|
|
570
|
+
|
|
571
|
+
Behaviour-preserving delegate: callers that do not ask for the failure
|
|
572
|
+
detail get exactly what they got before.
|
|
573
|
+
"""
|
|
574
|
+
pairs, _failures = connect_message_shards_detailed(
|
|
575
|
+
db_path, key_hex, salt_hex, passphrase)
|
|
576
|
+
return [conn for _path, conn in pairs]
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def count_talker_rows(conn, msg_table):
|
|
580
|
+
"""Rows this conversation has in one shard, or None if it has no table there.
|
|
581
|
+
|
|
582
|
+
Distinguishes "this shard holds nothing for the conversation" from "this
|
|
583
|
+
shard could not be read" - without it both look like zero.
|
|
584
|
+
"""
|
|
585
|
+
try:
|
|
586
|
+
cur = conn.execute(
|
|
587
|
+
"SELECT COUNT(*) FROM sqlite_master WHERE name=?", (msg_table,))
|
|
588
|
+
if cur.fetchone()[0] == 0:
|
|
589
|
+
return None
|
|
590
|
+
return conn.execute('SELECT COUNT(*) FROM "%s"' % msg_table).fetchone()[0]
|
|
591
|
+
except Exception:
|
|
592
|
+
return None
|
|
539
593
|
|
|
540
594
|
|
|
541
595
|
def msg_tables(conn):
|
|
@@ -808,12 +862,17 @@ def _message_dict(row, sender_id_map, name_map, own_wxid, is_group=False):
|
|
|
808
862
|
}
|
|
809
863
|
|
|
810
864
|
|
|
811
|
-
def get_messages(conns, talker, limit=100, offset=0, name_map=None, own_wxid=None
|
|
865
|
+
def get_messages(conns, talker, limit=100, offset=0, name_map=None, own_wxid=None,
|
|
866
|
+
shard_names=None, shard_report=None):
|
|
812
867
|
"""Get messages for a specific talker, merged across every shard.
|
|
813
868
|
|
|
814
869
|
Args:
|
|
815
870
|
name_map: optional {wxid: display_name} dict for resolving sender names
|
|
816
871
|
own_wxid: account owner wxid for self-message detection
|
|
872
|
+
shard_names: optional basename per connection, parallel to `conns`
|
|
873
|
+
shard_report: optional list to append one outcome dict per shard to.
|
|
874
|
+
Purely additive: the returned messages are identical with or
|
|
875
|
+
without it, which is asserted by test/nt_decrypt_shards_test.py.
|
|
817
876
|
"""
|
|
818
877
|
if name_map is None:
|
|
819
878
|
name_map = {}
|
|
@@ -821,19 +880,36 @@ def get_messages(conns, talker, limit=100, offset=0, name_map=None, own_wxid=Non
|
|
|
821
880
|
msg_table = f"Msg_{hashlib.md5(talker.encode()).hexdigest()}"
|
|
822
881
|
is_group = '@chatroom' in talker
|
|
823
882
|
|
|
883
|
+
def record(index, opened, has_table, rows_for_talker, reason):
|
|
884
|
+
if shard_report is None:
|
|
885
|
+
return
|
|
886
|
+
name = shard_names[index] if shard_names and index < len(shard_names) else ''
|
|
887
|
+
shard_report.append({
|
|
888
|
+
'name': name,
|
|
889
|
+
'opened': opened,
|
|
890
|
+
'hasTalkerTable': has_table,
|
|
891
|
+
'rowsForTalker': rows_for_talker,
|
|
892
|
+
'reason': reason,
|
|
893
|
+
})
|
|
894
|
+
|
|
824
895
|
# Each shard only needs to yield its newest window: once every shard's rows
|
|
825
896
|
# are merged and re-sorted, nothing older than that can reach this page.
|
|
826
897
|
window = 0 if limit <= 0 else limit + offset
|
|
827
898
|
collected = []
|
|
828
899
|
found = False
|
|
829
900
|
|
|
830
|
-
for conn in conns:
|
|
901
|
+
for index, conn in enumerate(conns):
|
|
831
902
|
c = conn.cursor()
|
|
832
903
|
try:
|
|
833
904
|
c.execute("SELECT COUNT(*) FROM sqlite_master WHERE name=?", (msg_table,))
|
|
834
905
|
if c.fetchone()[0] == 0:
|
|
906
|
+
# The shard is readable and simply holds nothing for this
|
|
907
|
+
# conversation - distinct from not being readable at all.
|
|
908
|
+
record(index, True, False, None, None)
|
|
835
909
|
continue
|
|
836
910
|
found = True
|
|
911
|
+
rows_for_talker = c.execute(
|
|
912
|
+
'SELECT COUNT(*) FROM "%s"' % msg_table).fetchone()[0]
|
|
837
913
|
|
|
838
914
|
sql = f'''
|
|
839
915
|
SELECT local_id, server_id, local_type, sort_seq, real_sender_id,
|
|
@@ -852,7 +928,11 @@ def get_messages(conns, talker, limit=100, offset=0, name_map=None, own_wxid=Non
|
|
|
852
928
|
c.execute("SELECT rowid, user_name FROM Name2Id")
|
|
853
929
|
sender_id_map = {rowid: uname for rowid, uname in c.fetchall()}
|
|
854
930
|
except Exception:
|
|
931
|
+
# This shard had the conversation's table but could not be read.
|
|
932
|
+
# Previously this was indistinguishable from "no rows here".
|
|
933
|
+
record(index, True, None, None, 'READ_FAILED')
|
|
855
934
|
continue
|
|
935
|
+
record(index, True, True, rows_for_talker, None)
|
|
856
936
|
|
|
857
937
|
for row in rows:
|
|
858
938
|
collected.append(_message_dict(row, sender_id_map, name_map, own_wxid, is_group))
|
|
@@ -1138,6 +1218,9 @@ def main():
|
|
|
1138
1218
|
msg_parser.add_argument('--contact-key', default=os.environ.get('WEFLOW_CONTACT_KEY'), help='Contact DB key hex (64 chars)')
|
|
1139
1219
|
msg_parser.add_argument('--contact-salt', default=os.environ.get('WEFLOW_CONTACT_SALT'), help='Contact DB salt hex (32 chars)')
|
|
1140
1220
|
msg_parser.add_argument('--own-wxid', default=os.environ.get('WEFLOW_OWN_WXID'), help='Account owner wxid (for self-message detection)')
|
|
1221
|
+
# Opt-in: without it the JSON is byte-identical to before this flag existed.
|
|
1222
|
+
msg_parser.add_argument('--report-shards', action='store_true',
|
|
1223
|
+
help='Add a per-shard read report to the result (additive)')
|
|
1141
1224
|
|
|
1142
1225
|
# contacts command
|
|
1143
1226
|
contacts_parser = sub.add_parser('contacts', help='List contacts')
|
|
@@ -1282,12 +1365,37 @@ def main():
|
|
|
1282
1365
|
conn.close()
|
|
1283
1366
|
|
|
1284
1367
|
elif args.command == 'messages':
|
|
1285
|
-
|
|
1286
|
-
if not conns:
|
|
1287
|
-
print(json.dumps({"error": "无法打开消息数据库,请检查密钥"}, ensure_ascii=True))
|
|
1288
|
-
return
|
|
1368
|
+
passphrase = getattr(args, 'passphrase', '') or ''
|
|
1289
1369
|
own_wxid = getattr(args, 'own_wxid', None)
|
|
1290
|
-
|
|
1370
|
+
|
|
1371
|
+
if getattr(args, 'report_shards', False):
|
|
1372
|
+
pairs, failures = connect_message_shards_detailed(
|
|
1373
|
+
args.db, args.key, args.salt, passphrase)
|
|
1374
|
+
if not pairs:
|
|
1375
|
+
print(json.dumps({"error": "无法打开消息数据库,请检查密钥"}, ensure_ascii=True))
|
|
1376
|
+
return
|
|
1377
|
+
conns = [conn for _path, conn in pairs]
|
|
1378
|
+
names = [os.path.basename(path) for path, _conn in pairs]
|
|
1379
|
+
shard_items = []
|
|
1380
|
+
result = get_messages(conns, args.talker, args.limit, args.offset,
|
|
1381
|
+
contact_name_map, own_wxid,
|
|
1382
|
+
shard_names=names, shard_report=shard_items)
|
|
1383
|
+
# Failure entries carry no opened connection, so they are not in
|
|
1384
|
+
# `conns` and could not be recorded by get_messages itself.
|
|
1385
|
+
result['shards'] = {
|
|
1386
|
+
'scanned': len(pairs) + len(failures),
|
|
1387
|
+
'opened': len(pairs),
|
|
1388
|
+
'failed': len(failures),
|
|
1389
|
+
'items': failures + shard_items,
|
|
1390
|
+
}
|
|
1391
|
+
else:
|
|
1392
|
+
conns = connect_message_shards(args.db, args.key, args.salt, passphrase)
|
|
1393
|
+
if not conns:
|
|
1394
|
+
print(json.dumps({"error": "无法打开消息数据库,请检查密钥"}, ensure_ascii=True))
|
|
1395
|
+
return
|
|
1396
|
+
result = get_messages(conns, args.talker, args.limit, args.offset,
|
|
1397
|
+
contact_name_map, own_wxid)
|
|
1398
|
+
|
|
1291
1399
|
print(json.dumps(result, ensure_ascii=True))
|
|
1292
1400
|
for conn in conns:
|
|
1293
1401
|
conn.close()
|
package/src/core/ntCore.ts
CHANGED
|
@@ -30,8 +30,28 @@ export interface NtSessionsResult extends NtResult {
|
|
|
30
30
|
sessions?: ChatSession[]
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
/** One shard's outcome, as reported by `nt_decrypt.py --report-shards`. */
|
|
34
|
+
export interface ShardOutcome {
|
|
35
|
+
/** basename only - the child never reports a full path */
|
|
36
|
+
name: string
|
|
37
|
+
opened: boolean
|
|
38
|
+
/** null when the shard could not be read at all */
|
|
39
|
+
hasTalkerTable: boolean | null
|
|
40
|
+
rowsForTalker: number | null
|
|
41
|
+
/** null | KEY_REJECTED | OPEN_FAILED | READ_FAILED */
|
|
42
|
+
reason: string | null
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface ShardReport {
|
|
46
|
+
scanned: number
|
|
47
|
+
opened: number
|
|
48
|
+
failed: number
|
|
49
|
+
items: ShardOutcome[]
|
|
50
|
+
}
|
|
51
|
+
|
|
33
52
|
export interface NtMessagesResult extends NtResult {
|
|
34
53
|
messages?: Message[]
|
|
54
|
+
shards?: ShardReport
|
|
35
55
|
}
|
|
36
56
|
|
|
37
57
|
export interface NtContactsResult extends NtResult {
|
|
@@ -282,6 +302,28 @@ export class NtCore {
|
|
|
282
302
|
return { success: true, messages: result.messages || [] }
|
|
283
303
|
}
|
|
284
304
|
|
|
305
|
+
/**
|
|
306
|
+
* `getMessages` plus the per-shard read report.
|
|
307
|
+
*
|
|
308
|
+
* Separate method rather than a flag on `getMessages`: the extra argv token
|
|
309
|
+
* changes what the child returns, and every existing caller is better off
|
|
310
|
+
* with the smaller payload it already parses.
|
|
311
|
+
*/
|
|
312
|
+
async getMessagesWithShards(talker: string, limit = 100, offset = 0): Promise<NtMessagesResult> {
|
|
313
|
+
const args: string[] = [
|
|
314
|
+
'messages',
|
|
315
|
+
'--limit', String(limit),
|
|
316
|
+
'--offset', String(offset),
|
|
317
|
+
'--report-shards',
|
|
318
|
+
]
|
|
319
|
+
const ownWxid = configService.get('wxid')
|
|
320
|
+
const result = await this.callPython(args, { talker, ownWxid })
|
|
321
|
+
if (result.error) {
|
|
322
|
+
return { success: false, error: result.error }
|
|
323
|
+
}
|
|
324
|
+
return { success: true, messages: result.messages || [], shards: result.shards }
|
|
325
|
+
}
|
|
326
|
+
|
|
285
327
|
async getContacts(keyword?: string, limit = 200): Promise<NtContactsResult> {
|
|
286
328
|
const args: string[] = [
|
|
287
329
|
'contacts',
|
|
@@ -3,6 +3,7 @@ import { existsSync } from 'fs'
|
|
|
3
3
|
import { WcdbCore } from '../core/wcdbCore.js'
|
|
4
4
|
import { SqlcipherCore } from '../core/sqlcipherCore.js'
|
|
5
5
|
import { NtCore } from '../core/ntCore.js'
|
|
6
|
+
import type { ShardReport } from '../core/ntCore.js'
|
|
6
7
|
import { configService } from './configService.js'
|
|
7
8
|
import type { ChatSession, Message, Contact, DataVersion } from '../types.js'
|
|
8
9
|
import { collectMessagesInRange } from './messageQuery.js'
|
|
@@ -239,6 +240,31 @@ export class ChatService {
|
|
|
239
240
|
return []
|
|
240
241
|
}
|
|
241
242
|
|
|
243
|
+
/**
|
|
244
|
+
* Messages plus a per-shard read report, when the backend can produce one.
|
|
245
|
+
*
|
|
246
|
+
* Only the NT path (Python) knows about shards. The legacy 3.x, raw-4.x and
|
|
247
|
+
* WCDB paths return `shards: undefined`, which callers must read as "this
|
|
248
|
+
* backend cannot tell you" rather than "everything was read" - see
|
|
249
|
+
* `coverage: 'unverified'` in docs/SYNC_CONTRACT.md.
|
|
250
|
+
*/
|
|
251
|
+
async getMessagesWithShards(
|
|
252
|
+
talker: string,
|
|
253
|
+
limit = 0,
|
|
254
|
+
offset = 0
|
|
255
|
+
): Promise<{ messages: Message[]; shards?: ShardReport }> {
|
|
256
|
+
const conn = await this.connect()
|
|
257
|
+
if (!conn.success) return { messages: [] }
|
|
258
|
+
|
|
259
|
+
if (this.ntCore) {
|
|
260
|
+
const result = await this.ntCore.getMessagesWithShards(talker, limit, offset)
|
|
261
|
+
if (!result.success || !result.messages) return { messages: [] }
|
|
262
|
+
return { messages: result.messages, shards: result.shards }
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return { messages: await this.getMessages(talker, limit, offset) }
|
|
266
|
+
}
|
|
267
|
+
|
|
242
268
|
async getMessagesInRange(
|
|
243
269
|
talker: string,
|
|
244
270
|
limit = 0,
|