mpdt 0.2.3__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mpdt-0.2.3 → mpdt-0.2.4}/PKG-INFO +1 -1
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/__init__.py +1 -1
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/cli.py +14 -2
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt.egg-info/PKG-INFO +1 -1
- {mpdt-0.2.3 → mpdt-0.2.4}/pyproject.toml +1 -1
- {mpdt-0.2.3 → mpdt-0.2.4}/tests/test_cli.py +39 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/LICENSE +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/README.md +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/__main__.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/config.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/errors.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/fetchers/__init__.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/fetchers/article_detail.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/fetchers/datacube.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/fetchers/publish_list.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/metrics.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/report.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/session.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt/store.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt.egg-info/SOURCES.txt +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt.egg-info/dependency_links.txt +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt.egg-info/entry_points.txt +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt.egg-info/requires.txt +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/mpdt.egg-info/top_level.txt +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/setup.cfg +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/tests/test_config.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/tests/test_fetchers.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/tests/test_metrics.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/tests/test_report.py +0 -0
- {mpdt-0.2.3 → mpdt-0.2.4}/tests/test_store.py +0 -0
|
@@ -100,9 +100,21 @@ def cmd_fetch(args):
|
|
|
100
100
|
pl = publish_list.fetch(s, count=args.count)
|
|
101
101
|
for i, a in enumerate(pl['articles'], 1):
|
|
102
102
|
r.progress(i, len(pl['articles']), f'入库 {a["title"][:24]}')
|
|
103
|
+
# R101:create_time/publish_date/send_hour 元数据随行入库(C20 publish_window
|
|
104
|
+
# 消费链的 L1 缺口——解析层早有、入库曾丢弃)。send_hour=群发时刻北京时间
|
|
105
|
+
# HH:MM;上游缺 create_time 时三键均存 null,不虚构。
|
|
106
|
+
ct = a.get('create_time')
|
|
107
|
+
send_hour = None
|
|
108
|
+
if isinstance(ct, int) and ct > 0:
|
|
109
|
+
send_hour = dt.datetime.fromtimestamp(
|
|
110
|
+
ct, dt.timezone(dt.timedelta(hours=8))).strftime('%H:%M')
|
|
103
111
|
store.upsert(con, 'publish_list', stat_date, a['appmsgid'], a['itemidx'],
|
|
104
|
-
a['title'], {k: a
|
|
105
|
-
|
|
112
|
+
a['title'], {**{k: a.get(k) for k in
|
|
113
|
+
('read_num', 'share_num', 'like_num',
|
|
114
|
+
'old_like_num', 'comment_num')},
|
|
115
|
+
'create_time': ct if isinstance(ct, int) and ct > 0 else None,
|
|
116
|
+
'publish_date': a.get('publish_date') or None,
|
|
117
|
+
'send_hour': send_hour})
|
|
106
118
|
n_total += len(pl['articles'])
|
|
107
119
|
store.log_run(con, 'fetch:publish_list', True, f"{len(pl['articles'])} articles")
|
|
108
120
|
r.say(f'拉取 datacube({begin} ~ {end},T+1 口径)…')
|
|
@@ -285,6 +285,45 @@ class TestFetchStubbed(CliBase):
|
|
|
285
285
|
con.close()
|
|
286
286
|
self.assertEqual(n, 2)
|
|
287
287
|
|
|
288
|
+
def test_fetch_persists_create_time_and_send_hour(self):
|
|
289
|
+
"""R101:publish_list 入库须带 create_time/publish_date/send_hour(元数据)。
|
|
290
|
+
|
|
291
|
+
根因=C20 external_rules.publish_window 结构性不可消费——解析层提取了
|
|
292
|
+
create_time 但 cli 入库只存 5 数值指标,发布时刻在 L1 就被丢弃。
|
|
293
|
+
send_hour 口径=群发时刻(publish_info.create_time,贴图回退 sent_info.time)
|
|
294
|
+
的北京时间 HH:MM。
|
|
295
|
+
"""
|
|
296
|
+
art = self.pl_article(create_time=1791508200, publish_date='2026-10-09') # 10-09 09:10+08(parse 层从 create_time 派生 publish_date,恒一致)
|
|
297
|
+
pl = _pl_stub([art])
|
|
298
|
+
dc = _dc_stub([])
|
|
299
|
+
rc, out, err = self.run_cli(['--json', 'fetch'], patch_fetch=(pl, dc))
|
|
300
|
+
self.assertEqual(rc, 0)
|
|
301
|
+
con = real_store.connect(config.DEFAULTS['db_path'])
|
|
302
|
+
row = con.execute(
|
|
303
|
+
"SELECT metrics FROM snapshots WHERE stat_date='2026-10-17'"
|
|
304
|
+
" AND source='publish_list' AND appmsgid='mA'").fetchone()
|
|
305
|
+
con.close()
|
|
306
|
+
self.assertIsNotNone(row, 'publish_list 行未入库')
|
|
307
|
+
m = json.loads(row[0])
|
|
308
|
+
self.assertEqual(m.get("create_time"), 1791508200)
|
|
309
|
+
self.assertEqual(m.get('publish_date'), '2026-10-09')
|
|
310
|
+
self.assertEqual(m.get('send_hour'), '09:10')
|
|
311
|
+
|
|
312
|
+
def test_fetch_persists_send_hour_none_when_no_create_time(self):
|
|
313
|
+
art = self.pl_article(create_time=None) # 老上游缺字段:不虚构,存 null
|
|
314
|
+
pl = _pl_stub([art])
|
|
315
|
+
dc = _dc_stub([])
|
|
316
|
+
rc, out, err = self.run_cli(['--json', 'fetch'], patch_fetch=(pl, dc))
|
|
317
|
+
self.assertEqual(rc, 0)
|
|
318
|
+
con = real_store.connect(config.DEFAULTS['db_path'])
|
|
319
|
+
row = con.execute(
|
|
320
|
+
"SELECT metrics FROM snapshots WHERE stat_date='2026-10-17'"
|
|
321
|
+
" AND source='publish_list' AND appmsgid='mA'").fetchone()
|
|
322
|
+
con.close()
|
|
323
|
+
m = json.loads(row[0])
|
|
324
|
+
self.assertIsNone(m.get('send_hour'))
|
|
325
|
+
self.assertIsNone(m.get('create_time'))
|
|
326
|
+
|
|
288
327
|
def test_fetch_from_to_dates(self):
|
|
289
328
|
pl = _pl_stub([])
|
|
290
329
|
dc = _dc_stub([])
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|