mdbq 2.2.3__py3-none-any.whl → 2.2.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/aggregation/query_data.py +155 -155
- mdbq/other/sku_picture.py +9 -4
- {mdbq-2.2.3.dist-info → mdbq-2.2.5.dist-info}/METADATA +1 -1
- {mdbq-2.2.3.dist-info → mdbq-2.2.5.dist-info}/RECORD +6 -6
- {mdbq-2.2.3.dist-info → mdbq-2.2.5.dist-info}/WHEEL +0 -0
- {mdbq-2.2.3.dist-info → mdbq-2.2.5.dist-info}/top_level.txt +0 -0
mdbq/aggregation/query_data.py
CHANGED
|
@@ -1599,120 +1599,120 @@ def data_aggregation(service_databases=[{}], months=1):
|
|
|
1599
1599
|
|
|
1600
1600
|
# 从数据库中获取数据, 返回包含 df 数据的字典
|
|
1601
1601
|
data_dict = [
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1602
|
+
{
|
|
1603
|
+
'数据库名': '聚合数据', # 清洗完回传的目的地数据库
|
|
1604
|
+
'集合名': '天猫_主体报表', # 清洗完回传的数据表名
|
|
1605
|
+
'唯一主键': ['日期', '推广渠道', '营销场景', '商品id', '花费'],
|
|
1606
|
+
'数据主体': sdq.tg_wxt(),
|
|
1607
|
+
},
|
|
1608
|
+
{
|
|
1609
|
+
'数据库名': '聚合数据',
|
|
1610
|
+
'集合名': '天猫生意经_宝贝指标',
|
|
1611
|
+
'唯一主键': ['日期', '宝贝id'], # 不能加其他字段做主键，比如销售额，是变动的，不是唯一的
|
|
1612
|
+
'数据主体': sdq.syj(),
|
|
1613
|
+
},
|
|
1614
|
+
{
|
|
1615
|
+
'数据库名': '聚合数据',
|
|
1616
|
+
'集合名': '天猫_店铺来源_日数据',
|
|
1617
|
+
'唯一主键': ['日期', '一级来源', '二级来源', '三级来源', '访客数'],
|
|
1618
|
+
'数据主体': sdq.dplyd(),
|
|
1619
|
+
},
|
|
1620
|
+
{
|
|
1621
|
+
'数据库名': '聚合数据',
|
|
1622
|
+
'集合名': '天猫_店铺来源_日数据_旧版',
|
|
1623
|
+
'唯一主键': ['日期', '一级来源', '二级来源', '三级来源'],
|
|
1624
|
+
'数据主体': sdq.dplyd_old(),
|
|
1625
|
+
},
|
|
1626
|
+
{
|
|
1627
|
+
'数据库名': '聚合数据',
|
|
1628
|
+
'集合名': '商品id编码表',
|
|
1629
|
+
'唯一主键': ['宝贝id'],
|
|
1630
|
+
'数据主体': sdq.idbm(),
|
|
1631
|
+
},
|
|
1632
|
+
{
|
|
1633
|
+
'数据库名': '聚合数据',
|
|
1634
|
+
'集合名': '商品id图片对照表',
|
|
1635
|
+
'唯一主键': ['商品id'],
|
|
1636
|
+
'数据主体': sdq.sp_picture(),
|
|
1637
|
+
},
|
|
1638
|
+
{
|
|
1639
|
+
'数据库名': '聚合数据',
|
|
1640
|
+
'集合名': '商品成本',
|
|
1641
|
+
'唯一主键': ['款号'],
|
|
1642
|
+
'数据主体': sdq.sp_cost(),
|
|
1643
|
+
},
|
|
1644
|
+
{
|
|
1645
|
+
'数据库名': '聚合数据',
|
|
1646
|
+
'集合名': '京东_京准通',
|
|
1647
|
+
'唯一主键': ['日期', '产品线', '触发sku id', '跟单sku id', '花费', ],
|
|
1648
|
+
'数据主体': sdq.jdjzt(),
|
|
1649
|
+
},
|
|
1650
|
+
{
|
|
1651
|
+
'数据库名': '聚合数据',
|
|
1652
|
+
'集合名': '京东_京准通_全站营销',
|
|
1653
|
+
'唯一主键': ['日期', '产品线', '花费'],
|
|
1654
|
+
'数据主体': sdq.jdqzyx(),
|
|
1655
|
+
},
|
|
1656
|
+
{
|
|
1657
|
+
'数据库名': '聚合数据',
|
|
1658
|
+
'集合名': '京东_sku_商品明细',
|
|
1659
|
+
'唯一主键': ['日期', '商品id', '成交单量'],
|
|
1660
|
+
'数据主体': sdq.sku_sales(),
|
|
1661
|
+
},
|
|
1662
|
+
{
|
|
1663
|
+
'数据库名': '聚合数据',
|
|
1664
|
+
'集合名': '京东_spu_商品明细',
|
|
1665
|
+
'唯一主键': ['日期', '商品id', '成交单量'],
|
|
1666
|
+
'数据主体': sdq.spu_sales(),
|
|
1667
|
+
},
|
|
1668
1668
|
{
|
|
1669
1669
|
'数据库名': '聚合数据',
|
|
1670
1670
|
'集合名': '天猫_人群报表',
|
|
1671
1671
|
'唯一主键': ['日期', '推广渠道', '营销场景', '商品id', '花费', '人群名字'],
|
|
1672
1672
|
'数据主体': sdq.tg_rqbb(),
|
|
1673
1673
|
},
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1674
|
+
{
|
|
1675
|
+
'数据库名': '聚合数据',
|
|
1676
|
+
'集合名': '天猫_关键词报表',
|
|
1677
|
+
'唯一主键': ['日期', '推广渠道', '营销场景', '商品id', '花费', '词类型', '词名字/词包名字',],
|
|
1678
|
+
'数据主体': sdq.tg_gjc(),
|
|
1679
|
+
},
|
|
1680
|
+
{
|
|
1681
|
+
'数据库名': '聚合数据',
|
|
1682
|
+
'集合名': '天猫_超级直播',
|
|
1683
|
+
'唯一主键': ['日期', '推广渠道', '营销场景', '花费'],
|
|
1684
|
+
'数据主体': sdq.tg_cjzb(),
|
|
1685
|
+
},
|
|
1686
|
+
{
|
|
1687
|
+
'数据库名': '聚合数据',
|
|
1688
|
+
'集合名': '京东_关键词报表',
|
|
1689
|
+
'唯一主键': ['日期', '产品线', '搜索词', '关键词', '展现数', '花费'],
|
|
1690
|
+
'数据主体': sdq.jd_gjc(),
|
|
1691
|
+
},
|
|
1692
|
+
{
|
|
1693
|
+
'数据库名': '聚合数据',
|
|
1694
|
+
'集合名': '天猫_品销宝账户报表',
|
|
1695
|
+
'唯一主键': ['日期', '报表类型', '推广渠道', '营销场景', '花费'],
|
|
1696
|
+
'数据主体': sdq.pxb_zh(),
|
|
1697
|
+
},
|
|
1698
|
+
{
|
|
1699
|
+
'数据库名': '聚合数据',
|
|
1700
|
+
'集合名': '天猫店铺来源_手淘搜索',
|
|
1701
|
+
'唯一主键': ['日期', '关键词', '访客数'],
|
|
1702
|
+
'数据主体': sdq.tm_search(),
|
|
1703
|
+
},
|
|
1704
|
+
{
|
|
1705
|
+
'数据库名': '聚合数据',
|
|
1706
|
+
'集合名': '生意参谋_直播场次分析',
|
|
1707
|
+
'唯一主键': ['场次id'],
|
|
1708
|
+
'数据主体': sdq.zb_ccfx(),
|
|
1709
|
+
},
|
|
1710
|
+
{
|
|
1711
|
+
'数据库名': '聚合数据',
|
|
1712
|
+
'集合名': '多店推广场景_按日聚合',
|
|
1713
|
+
'唯一主键': [],
|
|
1714
|
+
'数据主体': sdq.tg_by_day(),
|
|
1715
|
+
},
|
|
1716
1716
|
]
|
|
1717
1717
|
for items in data_dict: # 遍历返回结果
|
|
1718
1718
|
db_name, table_name, unique_key_list, df = items['数据库名'], items['集合名'], items['唯一主键'], items['数据主体']
|
|
@@ -1753,53 +1753,53 @@ def data_aggregation(service_databases=[{}], months=1):
|
|
|
1753
1753
|
icm_update=unique_key_list,
|
|
1754
1754
|
service_database=service_database,
|
|
1755
1755
|
) # 3. 回传数据库
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1756
|
+
res = g.performance(bb_tg=True) # 盈亏表，依赖其他表，单独做
|
|
1757
|
+
m.df_to_mysql(
|
|
1758
|
+
df=res,
|
|
1759
|
+
db_name='聚合数据',
|
|
1760
|
+
table_name='_全店商品销售',
|
|
1761
|
+
move_insert=True, # 先删除，再插入
|
|
1762
|
+
# df_sql=True,
|
|
1763
|
+
# drop_duplicates=False,
|
|
1764
|
+
# icm_update=['日期', '商品id'], # 设置唯一主键
|
|
1765
|
+
service_database=service_database,
|
|
1766
|
+
)
|
|
1767
|
+
res = g.performance(bb_tg=False) # 盈亏表，依赖其他表，单独做
|
|
1768
|
+
m.df_to_mysql(
|
|
1769
|
+
df=res,
|
|
1770
|
+
db_name='聚合数据',
|
|
1771
|
+
table_name='_推广商品销售',
|
|
1772
|
+
move_insert=True, # 先删除，再插入
|
|
1773
|
+
# df_sql=True,
|
|
1774
|
+
# drop_duplicates=False,
|
|
1775
|
+
# icm_update=['日期', '商品id'], # 设置唯一主键
|
|
1776
|
+
service_database=service_database,
|
|
1777
|
+
)
|
|
1778
|
+
|
|
1779
|
+
res = g.performance_concat(bb_tg=False) # 推广主体合并直播表，依赖其他表，单独做
|
|
1780
|
+
m.df_to_mysql(
|
|
1781
|
+
df=res,
|
|
1782
|
+
db_name='聚合数据',
|
|
1783
|
+
table_name='天猫_推广汇总',
|
|
1784
|
+
move_insert=True, # 先删除，再插入
|
|
1785
|
+
# df_sql=True,
|
|
1786
|
+
# drop_duplicates=False,
|
|
1787
|
+
# icm_update=['日期', '推广渠道', '营销场景', '商品id', '花费', '展现量', '点击量'], # 设置唯一主键
|
|
1788
|
+
service_database=service_database,
|
|
1789
|
+
)
|
|
1790
|
+
|
|
1791
|
+
|
|
1792
|
+
res = g.performance_jd(jd_tg=False) # 盈亏表，依赖其他表，单独做
|
|
1793
|
+
m.df_to_mysql(
|
|
1794
|
+
df=res,
|
|
1795
|
+
db_name='聚合数据',
|
|
1796
|
+
table_name='_京东_推广商品销售',
|
|
1797
|
+
move_insert=True, # 先删除，再插入
|
|
1798
|
+
# df_sql=True,
|
|
1799
|
+
# drop_duplicates=False,
|
|
1800
|
+
# icm_update=['日期', '跟单sku id', '货号', '花费'], # 设置唯一主键
|
|
1801
|
+
service_database=service_database,
|
|
1802
|
+
)
|
|
1803
1803
|
|
|
1804
1804
|
|
|
1805
1805
|
# 这里要注释掉，不然 copysh.py 可能有问题，这里主要修改配置文件，后续触发 home_lx 的 optimize_datas.py(有s)程序进行全局清理
|
mdbq/other/sku_picture.py
CHANGED
|
@@ -598,6 +598,8 @@ class DownloadPicture():
|
|
|
598
598
|
url = data[col_name]
|
|
599
599
|
# self.filename = f'{data['店铺名称']}_{data['商品id']}_{data['商家编码']}.jpg'
|
|
600
600
|
self.filename = f'{data['商品id']}_{data['商家编码']}.jpg'
|
|
601
|
+
# 清除特殊符号，避免无法创建文件
|
|
602
|
+
self.filename = re.sub(r'[\\/\u4e00-\u9fa5‘’“”【】\[\]{}、，,～~!！]', '_', self.filename)
|
|
601
603
|
if os.path.isfile(os.path.join(self.save_path, self.filename)):
|
|
602
604
|
self.finish_download.append(data['商品id'])
|
|
603
605
|
i += 1
|
|
@@ -609,10 +611,13 @@ class DownloadPicture():
|
|
|
609
611
|
print(f'正在下载: {i}/{num}, {data['商品id']}')
|
|
610
612
|
self.headers.update({'User-Agent': ua_sj.get_ua()})
|
|
611
613
|
res = requests.get(url, headers=self.headers) # 下载图片到内存
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
614
|
+
try:
|
|
615
|
+
# 保存图片到本地文件夹
|
|
616
|
+
with open(os.path.join(self.save_path, self.filename), 'wb') as f:
|
|
617
|
+
f.write(res.content)
|
|
618
|
+
self.finish_download.append(data['商品id'])
|
|
619
|
+
except Exception as e:
|
|
620
|
+
print(f'{self.filename}: {e}')
|
|
616
621
|
i += 1
|
|
617
622
|
time.sleep(0.5)
|
|
618
623
|
|
|
@@ -5,7 +5,7 @@ mdbq/aggregation/aggregation.py,sha256=98pECXV6yw7XSjoLnJBgHIQWM2s2aaB8ii5qNebAI
|
|
|
5
5
|
mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
|
|
6
6
|
mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
|
|
7
7
|
mdbq/aggregation/optimize_data.py,sha256=Wis40oL04M7E1pkvgNPjyVFAUe-zgjimjIVAikxYY8Y,4418
|
|
8
|
-
mdbq/aggregation/query_data.py,sha256=
|
|
8
|
+
mdbq/aggregation/query_data.py,sha256=Za8shm_I9ESzfYUVPRPOh8kk3yVWwvWAvwV0mFVx5mI,85340
|
|
9
9
|
mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
|
|
10
10
|
mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
|
|
11
11
|
mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
|
|
@@ -31,14 +31,14 @@ mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,
|
|
|
31
31
|
mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
|
32
32
|
mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
|
|
33
33
|
mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
|
|
34
|
-
mdbq/other/sku_picture.py,sha256=
|
|
34
|
+
mdbq/other/sku_picture.py,sha256=ZgWhdiZYfrjUwId8i73dDvWTfKnih8Z7xmrK7b8xDD8,48442
|
|
35
35
|
mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
|
|
36
36
|
mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
|
|
37
37
|
mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
|
|
38
38
|
mdbq/pbix/refresh_all.py,sha256=viOlLCmz9zg61Q2nzjgl8dChfQxnxRd1A_jmQMb2oDM,5918
|
|
39
39
|
mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
|
|
40
40
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
|
41
|
-
mdbq-2.2.
|
|
42
|
-
mdbq-2.2.
|
|
43
|
-
mdbq-2.2.
|
|
44
|
-
mdbq-2.2.
|
|
41
|
+
mdbq-2.2.5.dist-info/METADATA,sha256=_CM2rouJXjLHpbYyF1uN8LSd4jOCgmceNwfNLD2J1Sk,245
|
|
42
|
+
mdbq-2.2.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
|
43
|
+
mdbq-2.2.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
|
44
|
+
mdbq-2.2.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|