mpdt 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mpdt-0.2.2 → mpdt-0.2.4}/PKG-INFO +1 -1
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/__init__.py +1 -1
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/cli.py +14 -2
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/fetchers/publish_list.py +5 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt.egg-info/PKG-INFO +1 -1
- {mpdt-0.2.2 → mpdt-0.2.4}/pyproject.toml +1 -1
- {mpdt-0.2.2 → mpdt-0.2.4}/tests/test_cli.py +39 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/tests/test_fetchers.py +14 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/LICENSE +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/README.md +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/__main__.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/config.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/errors.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/fetchers/__init__.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/fetchers/article_detail.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/fetchers/datacube.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/metrics.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/report.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/session.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt/store.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt.egg-info/SOURCES.txt +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt.egg-info/dependency_links.txt +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt.egg-info/entry_points.txt +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt.egg-info/requires.txt +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/mpdt.egg-info/top_level.txt +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/setup.cfg +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/tests/test_config.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/tests/test_metrics.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/tests/test_report.py +0 -0
- {mpdt-0.2.2 → mpdt-0.2.4}/tests/test_store.py +0 -0
|
@@ -100,9 +100,21 @@ def cmd_fetch(args):
|
|
|
100
100
|
pl = publish_list.fetch(s, count=args.count)
|
|
101
101
|
for i, a in enumerate(pl['articles'], 1):
|
|
102
102
|
r.progress(i, len(pl['articles']), f'入库 {a["title"][:24]}')
|
|
103
|
+
# R101:create_time/publish_date/send_hour 元数据随行入库(C20 publish_window
|
|
104
|
+
# 消费链的 L1 缺口——解析层早有、入库曾丢弃)。send_hour=群发时刻北京时间
|
|
105
|
+
# HH:MM;上游缺 create_time 时三键均存 null,不虚构。
|
|
106
|
+
ct = a.get('create_time')
|
|
107
|
+
send_hour = None
|
|
108
|
+
if isinstance(ct, int) and ct > 0:
|
|
109
|
+
send_hour = dt.datetime.fromtimestamp(
|
|
110
|
+
ct, dt.timezone(dt.timedelta(hours=8))).strftime('%H:%M')
|
|
103
111
|
store.upsert(con, 'publish_list', stat_date, a['appmsgid'], a['itemidx'],
|
|
104
|
-
a['title'], {k: a
|
|
105
|
-
|
|
112
|
+
a['title'], {**{k: a.get(k) for k in
|
|
113
|
+
('read_num', 'share_num', 'like_num',
|
|
114
|
+
'old_like_num', 'comment_num')},
|
|
115
|
+
'create_time': ct if isinstance(ct, int) and ct > 0 else None,
|
|
116
|
+
'publish_date': a.get('publish_date') or None,
|
|
117
|
+
'send_hour': send_hour})
|
|
106
118
|
n_total += len(pl['articles'])
|
|
107
119
|
store.log_run(con, 'fetch:publish_list', True, f"{len(pl['articles'])} articles")
|
|
108
120
|
r.say(f'拉取 datacube({begin} ~ {end},T+1 口径)…')
|
|
@@ -53,6 +53,11 @@ def parse_publish_list(raw: str) -> dict:
|
|
|
53
53
|
if not isinstance(info, dict):
|
|
54
54
|
continue
|
|
55
55
|
ct = (info.get('publish_info') or {}).get('create_time')
|
|
56
|
+
if not (isinstance(ct, int) and ct > 0):
|
|
57
|
+
# 贴图类(type=9 群发图)publish_info 为空对象,群发时间在 sent_info.time
|
|
58
|
+
# (R70 实证:2247485215 sent_info.time=1792145145 = 2026-10-16 18:05:45+08,
|
|
59
|
+
# 与 publish-read.py 实查 create_time 一致;保持 article 侧口径=群发时刻)
|
|
60
|
+
ct = (info.get('sent_info') or {}).get('time')
|
|
56
61
|
pub_date = ''
|
|
57
62
|
if isinstance(ct, int) and ct > 0:
|
|
58
63
|
pub_date = _dt.datetime.fromtimestamp(
|
|
@@ -285,6 +285,45 @@ class TestFetchStubbed(CliBase):
|
|
|
285
285
|
con.close()
|
|
286
286
|
self.assertEqual(n, 2)
|
|
287
287
|
|
|
288
|
+
def test_fetch_persists_create_time_and_send_hour(self):
|
|
289
|
+
"""R101:publish_list 入库须带 create_time/publish_date/send_hour(元数据)。
|
|
290
|
+
|
|
291
|
+
根因=C20 external_rules.publish_window 结构性不可消费——解析层提取了
|
|
292
|
+
create_time 但 cli 入库只存 5 数值指标,发布时刻在 L1 就被丢弃。
|
|
293
|
+
send_hour 口径=群发时刻(publish_info.create_time,贴图回退 sent_info.time)
|
|
294
|
+
的北京时间 HH:MM。
|
|
295
|
+
"""
|
|
296
|
+
art = self.pl_article(create_time=1791508200, publish_date='2026-10-09') # 10-09 09:10+08(parse 层从 create_time 派生 publish_date,恒一致)
|
|
297
|
+
pl = _pl_stub([art])
|
|
298
|
+
dc = _dc_stub([])
|
|
299
|
+
rc, out, err = self.run_cli(['--json', 'fetch'], patch_fetch=(pl, dc))
|
|
300
|
+
self.assertEqual(rc, 0)
|
|
301
|
+
con = real_store.connect(config.DEFAULTS['db_path'])
|
|
302
|
+
row = con.execute(
|
|
303
|
+
"SELECT metrics FROM snapshots WHERE stat_date='2026-10-17'"
|
|
304
|
+
" AND source='publish_list' AND appmsgid='mA'").fetchone()
|
|
305
|
+
con.close()
|
|
306
|
+
self.assertIsNotNone(row, 'publish_list 行未入库')
|
|
307
|
+
m = json.loads(row[0])
|
|
308
|
+
self.assertEqual(m.get("create_time"), 1791508200)
|
|
309
|
+
self.assertEqual(m.get('publish_date'), '2026-10-09')
|
|
310
|
+
self.assertEqual(m.get('send_hour'), '09:10')
|
|
311
|
+
|
|
312
|
+
def test_fetch_persists_send_hour_none_when_no_create_time(self):
|
|
313
|
+
art = self.pl_article(create_time=None) # 老上游缺字段:不虚构,存 null
|
|
314
|
+
pl = _pl_stub([art])
|
|
315
|
+
dc = _dc_stub([])
|
|
316
|
+
rc, out, err = self.run_cli(['--json', 'fetch'], patch_fetch=(pl, dc))
|
|
317
|
+
self.assertEqual(rc, 0)
|
|
318
|
+
con = real_store.connect(config.DEFAULTS['db_path'])
|
|
319
|
+
row = con.execute(
|
|
320
|
+
"SELECT metrics FROM snapshots WHERE stat_date='2026-10-17'"
|
|
321
|
+
" AND source='publish_list' AND appmsgid='mA'").fetchone()
|
|
322
|
+
con.close()
|
|
323
|
+
m = json.loads(row[0])
|
|
324
|
+
self.assertIsNone(m.get('send_hour'))
|
|
325
|
+
self.assertIsNone(m.get('create_time'))
|
|
326
|
+
|
|
288
327
|
def test_fetch_from_to_dates(self):
|
|
289
328
|
pl = _pl_stub([])
|
|
290
329
|
dc = _dc_stub([])
|
|
@@ -28,6 +28,20 @@ class TestPublishList(unittest.TestCase):
|
|
|
28
28
|
for k in ('appmsgid', 'itemidx', 'title', 'read_num', 'share_num', 'like_num'):
|
|
29
29
|
self.assertIn(k, a)
|
|
30
30
|
|
|
31
|
+
def test_pic_type_gets_publish_date_from_sent_info(self):
|
|
32
|
+
# R70:贴图类 publish_info 为空对象,时间戳在 sent_info.time(群发时间)。
|
|
33
|
+
# 实证:2247485215 sent_info.time=1792145145 → 2026-10-16 18:05:45+08,
|
|
34
|
+
# 与 PIC-rea 实查 t0(publish-read.py create_time)完全一致。
|
|
35
|
+
out = parse_publish_list(self.raw)
|
|
36
|
+
by_id = {a['appmsgid']: a for a in out['articles']}
|
|
37
|
+
a = by_id.get(2247485215)
|
|
38
|
+
self.assertIsNotNone(a, 'fixture 应含贴图样本 2247485215')
|
|
39
|
+
self.assertEqual(a['publish_date'], '2026-10-16')
|
|
40
|
+
self.assertEqual(a['create_time'], 1792145145)
|
|
41
|
+
b = by_id.get(2247485156)
|
|
42
|
+
self.assertIsNotNone(b, 'fixture 应含贴图样本 2247485156')
|
|
43
|
+
self.assertEqual(b['publish_date'], '2026-10-16')
|
|
44
|
+
|
|
31
45
|
def test_structure_change_raises_upstream_4(self):
|
|
32
46
|
with self.assertRaises(UpstreamError):
|
|
33
47
|
parse_publish_list('{"base_resp":{"ret":0},"publish_page":"not json"}')
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|