mdbq 4.0.55__py3-none-any.whl → 4.0.56__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/__version__.py +1 -1
- mdbq/mysql/s_query.py +22 -39
- {mdbq-4.0.55.dist-info → mdbq-4.0.56.dist-info}/METADATA +1 -1
- {mdbq-4.0.55.dist-info → mdbq-4.0.56.dist-info}/RECORD +6 -6
- {mdbq-4.0.55.dist-info → mdbq-4.0.56.dist-info}/WHEEL +0 -0
- {mdbq-4.0.55.dist-info → mdbq-4.0.56.dist-info}/top_level.txt +0 -0
mdbq/__version__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
VERSION = '4.0.
|
|
1
|
+
VERSION = '4.0.56'
|
mdbq/mysql/s_query.py
CHANGED
|
@@ -695,7 +695,7 @@ class QueryDatas:
|
|
|
695
695
|
|
|
696
696
|
def _build_query_sql(self, db_name: str, table_name: str, selected_columns: List[str],
|
|
697
697
|
date_column: Optional[str] = None, start_date: Optional[str] = None,
|
|
698
|
-
end_date: Optional[str] = None, limit: Optional[int] = None) -> Tuple[str, List[Any]]:
|
|
698
|
+
end_date: Optional[str] = None, limit: Optional[int] = None, order_by: Optional[str] = None) -> Tuple[str, List[Any]]:
|
|
699
699
|
"""
|
|
700
700
|
构建SQL查询语句和参数。
|
|
701
701
|
|
|
@@ -707,6 +707,7 @@ class QueryDatas:
|
|
|
707
707
|
start_date: 开始日期
|
|
708
708
|
end_date: 结束日期
|
|
709
709
|
limit: 限制返回行数，None表示不限制
|
|
710
|
+
order_by: 排序字段名
|
|
710
711
|
|
|
711
712
|
Returns:
|
|
712
713
|
SQL语句和参数列表的元组
|
|
@@ -748,6 +749,11 @@ class QueryDatas:
|
|
|
748
749
|
if conditions:
|
|
749
750
|
base_sql += " WHERE " + " AND ".join(conditions)
|
|
750
751
|
|
|
752
|
+
# 默认排序
|
|
753
|
+
if not order_by:
|
|
754
|
+
order_by = 'id'
|
|
755
|
+
base_sql += f" ORDER BY `{order_by}`"
|
|
756
|
+
|
|
751
757
|
# 只在显式指定limit时添加限制
|
|
752
758
|
if limit is not None:
|
|
753
759
|
if not isinstance(limit, int) or limit <= 0:
|
|
@@ -910,7 +916,7 @@ class QueryDatas:
|
|
|
910
916
|
# 构建基础SQL
|
|
911
917
|
base_sql, params, param_names = self._build_query_sql(
|
|
912
918
|
db_name, table_name, selected_columns,
|
|
913
|
-
date_column, start_date, end_date, None
|
|
919
|
+
date_column, start_date, end_date, None, order_by=None
|
|
914
920
|
)
|
|
915
921
|
|
|
916
922
|
# 如果指定了limit且没有指定page_size，使用简单查询
|
|
@@ -945,11 +951,7 @@ class QueryDatas:
|
|
|
945
951
|
# 分页查询
|
|
946
952
|
offset = 0
|
|
947
953
|
all_results = []
|
|
948
|
-
min_size, max_size = 1000, 10000
|
|
949
|
-
target_time = 1.0 # 期望每批1秒
|
|
950
|
-
|
|
951
954
|
while offset < total_count:
|
|
952
|
-
_p_time = time.time()
|
|
953
955
|
# 添加分页参数
|
|
954
956
|
page_sql = f"{base_sql} LIMIT %s OFFSET %s"
|
|
955
957
|
page_params = list(params) + [page_size, offset]
|
|
@@ -959,45 +961,26 @@ class QueryDatas:
|
|
|
959
961
|
if not page_results:
|
|
960
962
|
break
|
|
961
963
|
|
|
962
|
-
|
|
963
|
-
all_results.extend(page_results)
|
|
964
|
-
else:
|
|
965
|
-
if len(all_results) == 0:
|
|
966
|
-
all_results = pd.DataFrame(page_results)
|
|
967
|
-
else:
|
|
968
|
-
all_results = pd.concat([all_results, pd.DataFrame(page_results)], ignore_index=True)
|
|
969
|
-
|
|
970
|
-
duration = time.time() - _p_time
|
|
971
|
-
page_size = self._adjust_page_size(duration, page_size, min_size, max_size, target_time)
|
|
964
|
+
all_results.extend(page_results)
|
|
972
965
|
offset += len(page_results)
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
'表': table_name,
|
|
976
|
-
# '当前偏移量': offset,
|
|
977
|
-
'总记录数': total_count,
|
|
978
|
-
'已获取记录数': len(all_results) if return_format == 'list_dict' else len(all_results.index),
|
|
979
|
-
'本批耗时': f'{duration:.2f}',
|
|
980
|
-
'下批page_size': page_size
|
|
981
|
-
})
|
|
966
|
+
if len(page_results) < page_size:
|
|
967
|
+
break # 最后一页
|
|
982
968
|
|
|
983
|
-
|
|
984
|
-
all_results = self._convert_decimal_columns(all_results)
|
|
985
|
-
logger.info('查询完成', {
|
|
969
|
+
logger.info('分页查询完成', {
|
|
986
970
|
'库': db_name,
|
|
987
971
|
'表': table_name,
|
|
988
972
|
'总记录数': total_count,
|
|
989
|
-
'已获取记录数': len(all_results)
|
|
973
|
+
'已获取记录数': len(all_results),
|
|
990
974
|
'查询耗时': f'{time.time() - start_time:.2f}s',
|
|
991
|
-
'查询参数': {
|
|
992
|
-
'开始日期': start_date,
|
|
993
|
-
'结束日期': end_date,
|
|
994
|
-
'日期字段': date_column,
|
|
995
|
-
'限制行数': limit,
|
|
996
|
-
'分页大小': page_size,
|
|
997
|
-
'返回数据格式': return_format,
|
|
998
|
-
}
|
|
999
975
|
})
|
|
1000
|
-
|
|
976
|
+
|
|
977
|
+
if return_format == 'list_dict':
|
|
978
|
+
return all_results
|
|
979
|
+
else:
|
|
980
|
+
df = pd.DataFrame(all_results)
|
|
981
|
+
if not df.empty:
|
|
982
|
+
df = self._convert_decimal_columns(df)
|
|
983
|
+
return df
|
|
1001
984
|
|
|
1002
985
|
except Exception as e:
|
|
1003
986
|
logger.error('数据查询失败', {
|
|
@@ -1034,4 +1017,4 @@ def main():
|
|
|
1034
1017
|
|
|
1035
1018
|
if __name__ == '__main__':
|
|
1036
1019
|
main()
|
|
1037
|
-
pass
|
|
1020
|
+
pass
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
|
|
2
|
-
mdbq/__version__.py,sha256=
|
|
2
|
+
mdbq/__version__.py,sha256=gH8F1h5cU2xqNuh1vnRF2IC8i2q8gFo5ofCs5vKSQ58,18
|
|
3
3
|
mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
|
|
4
4
|
mdbq/log/mylogger.py,sha256=kPe3wsQNaB1slfX-Z7VMqzZoMoqPfc7ylYXZDBeFzzI,24945
|
|
5
5
|
mdbq/myconf/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
|
@@ -7,7 +7,7 @@ mdbq/myconf/myconf.py,sha256=rHvQCnQRKhQ49AZBke-Z4v28hyOLmHt4MylIuB0H6yA,33516
|
|
|
7
7
|
mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
|
|
8
8
|
mdbq/mysql/deduplicator.py,sha256=AB3gL7ZwhcmzGHSu4UY4M6YZVPFZ2wlAN3BCcwAhegQ,73074
|
|
9
9
|
mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
|
|
10
|
-
mdbq/mysql/s_query.py,sha256=
|
|
10
|
+
mdbq/mysql/s_query.py,sha256=3sPkYXWBmGGHtTbcayC1PD9d2CCKKq3q-FcEUzRXzTY,45497
|
|
11
11
|
mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
|
|
12
12
|
mdbq/mysql/uploader.py,sha256=dJRAtj-IebPgOHvKKeu-CqoeqfSaRqusSBgt_hXgNdw,88926
|
|
13
13
|
mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
|
@@ -22,7 +22,7 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
|
|
|
22
22
|
mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
|
|
23
23
|
mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
|
|
24
24
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
|
25
|
-
mdbq-4.0.
|
|
26
|
-
mdbq-4.0.
|
|
27
|
-
mdbq-4.0.
|
|
28
|
-
mdbq-4.0.
|
|
25
|
+
mdbq-4.0.56.dist-info/METADATA,sha256=rWRbLyWzJVprn5VKD0tHVhtwYSdhK7krNTKPqc4bGC4,364
|
|
26
|
+
mdbq-4.0.56.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
|
|
27
|
+
mdbq-4.0.56.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
|
28
|
+
mdbq-4.0.56.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|