funcguard 0.2.54__tar.gz → 0.2.56__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {funcguard-0.2.54 → funcguard-0.2.56}/PKG-INFO +13 -3
- {funcguard-0.2.54 → funcguard-0.2.56}/README.md +12 -2
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/__init__.py +6 -0
- funcguard-0.2.56/funcguard/data_models/request_models.py +11 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/ip_utils.py +2 -1
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/log_utils.py +21 -21
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/pd_utils/__init__.py +4 -3
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/pd_utils/convert_utils.py +140 -98
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/pd_utils/date_utils.py +6 -6
- funcguard-0.2.54/funcguard/pd_utils/fill_round.py → funcguard-0.2.56/funcguard/pd_utils/fill_utils.py +10 -35
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/pd_utils/filter.py +1 -2
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/pd_utils/statistics/agg_utils.py +6 -6
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/pd_utils/statistics/count_utils.py +10 -9
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/pd_utils/statistics/df_statistics.py +27 -22
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/pd_utils/statistics/mask_utils.py +13 -9
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/tools.py +16 -9
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard.egg-info/PKG-INFO +13 -3
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard.egg-info/SOURCES.txt +1 -1
- {funcguard-0.2.54 → funcguard-0.2.56}/setup.py +1 -1
- funcguard-0.2.54/funcguard/data_models/request_models.py +0 -12
- {funcguard-0.2.54 → funcguard-0.2.56}/LICENSE +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/calculate.py +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/core.py +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/data_models/__init__.py +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/pd_utils/json_utils/__init__.py +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/pd_utils/json_utils/json_parser.py +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/pd_utils/statistics/__init__.py +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/printer.py +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard/time_utils.py +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard.egg-info/dependency_links.txt +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard.egg-info/not-zip-safe +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard.egg-info/requires.txt +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/funcguard.egg-info/top_level.txt +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/setup.cfg +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/tests/__init__.py +0 -0
- {funcguard-0.2.54 → funcguard-0.2.56}/tests/test_pd_filter_empty.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: funcguard
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.56
|
|
4
4
|
Summary: FuncGuard是一个Python库,提供函数执行超时控制、重试机制、HTTP请求封装和格式化打印工具。
|
|
5
5
|
Home-page: https://github.com/tinycen/funcguard
|
|
6
6
|
Author: tinycen
|
|
@@ -188,6 +188,15 @@ response = send_request(
|
|
|
188
188
|
)
|
|
189
189
|
print(response)
|
|
190
190
|
|
|
191
|
+
# 带 URL 查询参数的 GET 请求
|
|
192
|
+
response = send_request(
|
|
193
|
+
method="GET",
|
|
194
|
+
url="https://api.example.com/users",
|
|
195
|
+
params={"page": 1, "size": 20, "search": "张三"}
|
|
196
|
+
# 等价于: https://api.example.com/users?page=1&size=20&search=%E5%BC%A0%E4%B8%89
|
|
197
|
+
)
|
|
198
|
+
print(response)
|
|
199
|
+
|
|
191
200
|
# POST 请求(自动将 dict/list 转为 JSON)
|
|
192
201
|
response = send_request(
|
|
193
202
|
method="POST",
|
|
@@ -515,7 +524,7 @@ FuncGuard提供了丰富的pandas数据处理功能,包括数据填充、类
|
|
|
515
524
|
|
|
516
525
|
```python
|
|
517
526
|
import pandas as pd
|
|
518
|
-
from funcguard.pd_utils import fill_na, convert_columns,
|
|
527
|
+
from funcguard.pd_utils import fill_na, convert_columns, load_json
|
|
519
528
|
|
|
520
529
|
# 快速示例
|
|
521
530
|
df = pd.DataFrame({
|
|
@@ -622,7 +631,8 @@ print(f"当前价格: {current_price}, 变化: {price_change}") # 输出: 当
|
|
|
622
631
|
| `pd_group_agg` | 分组聚合统计 | [查看](docs/pandas/agg.md) |
|
|
623
632
|
| `pd_build_mask` / `pd_build_masks` / `pd_combine_masks` | 掩码构建 | [查看](docs/pandas/mask.md) |
|
|
624
633
|
| `DataFrameStatistics` | 统计分析 | [查看](docs/pandas/statistics.md) |
|
|
625
|
-
| `pd_cal_date_diff`
|
|
634
|
+
| `pd_cal_date_diff` | 日期计算 | [查看](docs/pandas/date.md) |
|
|
635
|
+
| `pd_round_columns` | 数值舍入 | [查看](docs/pandas/convert.md) |
|
|
626
636
|
|
|
627
637
|
### 计算工具
|
|
628
638
|
|
|
@@ -156,6 +156,15 @@ response = send_request(
|
|
|
156
156
|
)
|
|
157
157
|
print(response)
|
|
158
158
|
|
|
159
|
+
# 带 URL 查询参数的 GET 请求
|
|
160
|
+
response = send_request(
|
|
161
|
+
method="GET",
|
|
162
|
+
url="https://api.example.com/users",
|
|
163
|
+
params={"page": 1, "size": 20, "search": "张三"}
|
|
164
|
+
# 等价于: https://api.example.com/users?page=1&size=20&search=%E5%BC%A0%E4%B8%89
|
|
165
|
+
)
|
|
166
|
+
print(response)
|
|
167
|
+
|
|
159
168
|
# POST 请求(自动将 dict/list 转为 JSON)
|
|
160
169
|
response = send_request(
|
|
161
170
|
method="POST",
|
|
@@ -483,7 +492,7 @@ FuncGuard提供了丰富的pandas数据处理功能,包括数据填充、类
|
|
|
483
492
|
|
|
484
493
|
```python
|
|
485
494
|
import pandas as pd
|
|
486
|
-
from funcguard.pd_utils import fill_na, convert_columns,
|
|
495
|
+
from funcguard.pd_utils import fill_na, convert_columns, load_json
|
|
487
496
|
|
|
488
497
|
# 快速示例
|
|
489
498
|
df = pd.DataFrame({
|
|
@@ -590,7 +599,8 @@ print(f"当前价格: {current_price}, 变化: {price_change}") # 输出: 当
|
|
|
590
599
|
| `pd_group_agg` | 分组聚合统计 | [查看](docs/pandas/agg.md) |
|
|
591
600
|
| `pd_build_mask` / `pd_build_masks` / `pd_combine_masks` | 掩码构建 | [查看](docs/pandas/mask.md) |
|
|
592
601
|
| `DataFrameStatistics` | 统计分析 | [查看](docs/pandas/statistics.md) |
|
|
593
|
-
| `pd_cal_date_diff`
|
|
602
|
+
| `pd_cal_date_diff` | 日期计算 | [查看](docs/pandas/date.md) |
|
|
603
|
+
| `pd_round_columns` | 数值舍入 | [查看](docs/pandas/convert.md) |
|
|
594
604
|
|
|
595
605
|
### 计算工具
|
|
596
606
|
|
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
try:
|
|
2
|
+
from importlib.metadata import version as _get_version
|
|
3
|
+
__version__ = _get_version("funcguard")
|
|
4
|
+
except Exception:
|
|
5
|
+
__version__ = "unknown"
|
|
6
|
+
|
|
1
7
|
from .core import timeout_handler, retry_function, ask_select
|
|
2
8
|
from .tools import send_request, curl_cffi_request, check_url_valid, encode_basic_auth, md5_hash
|
|
3
9
|
from .time_utils import (
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@dataclass
|
|
5
|
+
class RequestLog:
|
|
6
|
+
save_method: bool | None = True
|
|
7
|
+
save_url: bool | None = True
|
|
8
|
+
save_headers: bool | None = True
|
|
9
|
+
save_body: bool | None = True
|
|
10
|
+
save_response: bool | None = True
|
|
11
|
+
save_path: str | None = ""
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
import sys
|
|
5
|
-
from typing import
|
|
5
|
+
from typing import TextIO, cast
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
class ColoredFormatter(logging.Formatter):
|
|
@@ -92,7 +92,7 @@ if not hasattr(logging.Logger, "progress"):
|
|
|
92
92
|
setattr(logging.Logger, "progress", _logger_progress)
|
|
93
93
|
|
|
94
94
|
|
|
95
|
-
def _normalize_level(level:
|
|
95
|
+
def _normalize_level(level: int | str) -> int:
|
|
96
96
|
"""
|
|
97
97
|
规范化日志等级。
|
|
98
98
|
|
|
@@ -113,32 +113,32 @@ def _normalize_level(level: Union[int, str]) -> int:
|
|
|
113
113
|
|
|
114
114
|
|
|
115
115
|
def setup_logger(
|
|
116
|
-
name:
|
|
117
|
-
level:
|
|
118
|
-
stream:
|
|
116
|
+
name: str | None = None,
|
|
117
|
+
level: int | str = logging.DEBUG,
|
|
118
|
+
stream: TextIO | None = None,
|
|
119
119
|
message_only: bool = False,
|
|
120
120
|
) -> SuccessLogger:
|
|
121
121
|
"""
|
|
122
122
|
创建并配置彩色日志输出。
|
|
123
123
|
|
|
124
124
|
Args:
|
|
125
|
-
name: logger
|
|
125
|
+
name: logger 的名称,默认全局共享1个 logger 实例。
|
|
126
126
|
示例:
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
127
|
+
# 在 network.py
|
|
128
|
+
logger_a = setup_logger("network")
|
|
129
|
+
logger_a.debug("网络调试信息")
|
|
130
|
+
|
|
131
|
+
# 在 db.py
|
|
132
|
+
logger_b = setup_logger("db")
|
|
133
|
+
logger_b.debug("数据库调试信息")
|
|
134
|
+
|
|
135
|
+
作用:通过不同名称创建的 logger 互不干扰,适合在大型项目中使用。
|
|
136
|
+
每个 logger 设置不同的日志级别、格式、输出流等。
|
|
137
|
+
level: 日志等级。支持 int 或字符串,默认 "DEBUG"。根据等级过滤后交给 handler 输出。
|
|
138
|
+
常用等级:DEBUG(10), INFO(20), SUCCESS(25), WARNING(30), PROGRESS(35), ERROR(40), CRITICAL(50)。
|
|
139
|
+
字符串支持:"DEBUG"、"INFO"、"PROGRESS"、"SUCCESS"、"WARNING"/"WARN"、"ERROR"、"CRITICAL"/"FATAL"(大小写不敏感)。
|
|
140
|
+
stream: 输出流,默认 sys.stdout。
|
|
141
|
+
message_only: 是否仅输出日志消息(不包含时间与等级),默认 False。
|
|
142
142
|
|
|
143
143
|
Returns:
|
|
144
144
|
配置完成的 logger。示例:
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import pandas as pd
|
|
2
|
-
from .
|
|
2
|
+
from .fill_utils import fill_na
|
|
3
3
|
from .date_utils import fill_nat, cal_date_diff
|
|
4
4
|
from .convert_utils import (
|
|
5
5
|
convert_series,
|
|
@@ -8,7 +8,8 @@ from .convert_utils import (
|
|
|
8
8
|
convert_numeric_series,
|
|
9
9
|
load_json,
|
|
10
10
|
convert_str_datetime,
|
|
11
|
-
convert_datetime_str
|
|
11
|
+
convert_datetime_str,
|
|
12
|
+
round_columns
|
|
12
13
|
)
|
|
13
14
|
from .statistics import (
|
|
14
15
|
pd_build_mask,
|
|
@@ -29,7 +30,6 @@ __all__ = [
|
|
|
29
30
|
# 数据填充类
|
|
30
31
|
'fill_na',
|
|
31
32
|
'fill_nat',
|
|
32
|
-
'round_columns',
|
|
33
33
|
'cal_date_diff',
|
|
34
34
|
|
|
35
35
|
# 数据转换类
|
|
@@ -40,6 +40,7 @@ __all__ = [
|
|
|
40
40
|
'convert_str_datetime',
|
|
41
41
|
'convert_datetime_str',
|
|
42
42
|
'load_json',
|
|
43
|
+
'round_columns',
|
|
43
44
|
|
|
44
45
|
# 数据筛选类
|
|
45
46
|
"pd_filter",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import pandas as pd
|
|
2
2
|
from decimal import Decimal
|
|
3
|
-
from typing import
|
|
3
|
+
from typing import Any, Literal
|
|
4
4
|
from pandas import (
|
|
5
5
|
Int64Dtype,
|
|
6
6
|
Float64Dtype,
|
|
@@ -10,10 +10,55 @@ from pandas import (
|
|
|
10
10
|
from .json_utils import json_loads
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
# 数据类型映射常量
|
|
14
|
+
TYPE_MAPPING = {
|
|
15
|
+
"int": Int64Dtype(),
|
|
16
|
+
"float": Float64Dtype(),
|
|
17
|
+
"str": StringDtype(),
|
|
18
|
+
"bool": BooleanDtype(),
|
|
19
|
+
"datetime": "datetime64[ns]", # datetime使用字符串形式
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def convert_columns(df: pd.DataFrame, columns: dict[str, str], decimal_places: int | None = None) -> pd.DataFrame:
|
|
24
|
+
"""
|
|
25
|
+
转换DataFrame中指定列的数据类型。
|
|
26
|
+
|
|
27
|
+
参数:
|
|
28
|
+
- df (pd.DataFrame):输入的DataFrame。
|
|
29
|
+
- columns (Dict[str, str]):
|
|
30
|
+
要转换类型的字典,键为列名,值为目标数据类型。
|
|
31
|
+
支持的数据类型:'int', 'float', 'str', 'bool', 'datetime'。
|
|
32
|
+
- decimal_places (int, optional):当转换为'float'类型时保留的小数位数,默认为None表示不限制
|
|
33
|
+
|
|
34
|
+
返回:
|
|
35
|
+
- pd.DataFrame:列类型转换后的DataFrame。
|
|
36
|
+
"""
|
|
37
|
+
for column, target_type in columns.items():
|
|
38
|
+
if column in df.columns and target_type in TYPE_MAPPING:
|
|
39
|
+
try:
|
|
40
|
+
if target_type == "datetime":
|
|
41
|
+
df[column] = pd.to_datetime(df[column], errors="coerce")
|
|
42
|
+
elif target_type == "float":
|
|
43
|
+
df[column] = df[column].astype(
|
|
44
|
+
TYPE_MAPPING[target_type]
|
|
45
|
+
) # pyright: ignore[reportArgumentType]
|
|
46
|
+
if decimal_places is not None:
|
|
47
|
+
df[column] = df[column].round(decimal_places)
|
|
48
|
+
else:
|
|
49
|
+
df[column] = df[column].astype(
|
|
50
|
+
TYPE_MAPPING[target_type]
|
|
51
|
+
) # pyright: ignore[reportArgumentType]
|
|
52
|
+
except (ValueError, TypeError):
|
|
53
|
+
# 如果转换失败,保持原类型
|
|
54
|
+
pass
|
|
55
|
+
return df
|
|
56
|
+
|
|
57
|
+
|
|
13
58
|
def convert_series(
|
|
14
59
|
data: pd.Series,
|
|
15
60
|
return_type: Literal["dict", "df", "series"] = "dict"
|
|
16
|
-
) ->
|
|
61
|
+
) -> dict[Any, int | float] | pd.DataFrame | pd.Series:
|
|
17
62
|
"""
|
|
18
63
|
将 pandas Series 格式化为指定的返回类型。
|
|
19
64
|
|
|
@@ -52,29 +97,20 @@ def convert_series(
|
|
|
52
97
|
return data
|
|
53
98
|
|
|
54
99
|
|
|
55
|
-
|
|
56
|
-
TYPE_MAPPING = {
|
|
57
|
-
"int": Int64Dtype(),
|
|
58
|
-
"float": Float64Dtype(),
|
|
59
|
-
"str": StringDtype(),
|
|
60
|
-
"bool": BooleanDtype(),
|
|
61
|
-
"datetime": "datetime64[ns]", # datetime使用字符串形式
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def convert_numeric_series(series: pd.Series, decimal_places: Optional[int] = None) -> pd.Series:
|
|
100
|
+
def convert_numeric_series(series: pd.Series, decimal_places: int | None = None) -> pd.Series:
|
|
66
101
|
"""
|
|
67
102
|
将Series转换为数值类型,并自动检测应该使用 int 还是 float。
|
|
103
|
+
当 decimal_places == 0 时,round后自动转为Int64类型。
|
|
68
104
|
|
|
69
105
|
流程:
|
|
70
106
|
1. 先统一转换为 float64(确保所有值都是数值类型)
|
|
71
|
-
2.
|
|
72
|
-
3.
|
|
73
|
-
4.
|
|
107
|
+
2. 如果指定了decimal_places,先执行round操作
|
|
108
|
+
3. 当decimal_places为0时,round后自动转为Int64类型
|
|
109
|
+
4. 否则检测非空值中是否存在小数,没有小数则转换为Int64,否则保持float64
|
|
74
110
|
|
|
75
111
|
参数:
|
|
76
112
|
- series (pd.Series):输入的Series
|
|
77
|
-
- decimal_places (int, optional)
|
|
113
|
+
- decimal_places (int, optional):保留的小数位数,默认为None表示不限制
|
|
78
114
|
|
|
79
115
|
返回:
|
|
80
116
|
- pd.Series:转换后的Series
|
|
@@ -82,55 +118,87 @@ def convert_numeric_series(series: pd.Series, decimal_places: Optional[int] = No
|
|
|
82
118
|
# 先统一转换为 float
|
|
83
119
|
series = series.astype(float)
|
|
84
120
|
|
|
121
|
+
# 如果指定了decimal_places,先执行round操作
|
|
122
|
+
if decimal_places is not None:
|
|
123
|
+
series = series.round(decimal_places)
|
|
124
|
+
if decimal_places == 0:
|
|
125
|
+
# decimal_places为0时,round后自动转为Int64类型(支持NaN)
|
|
126
|
+
return series.astype("Int64")
|
|
127
|
+
|
|
85
128
|
# 检测是否存在小数
|
|
86
129
|
non_null_values = series.dropna()
|
|
87
130
|
if len(non_null_values) > 0:
|
|
88
131
|
# 检查是否有任何值有小数部分
|
|
89
132
|
has_decimal = any(non_null_values != non_null_values.round())
|
|
90
133
|
if not has_decimal:
|
|
91
|
-
# 没有小数,转换为
|
|
92
|
-
return series.astype("
|
|
134
|
+
# 没有小数,转换为 Int64(支持NaN),并返回Int64类型的Series
|
|
135
|
+
return series.astype("Int64")
|
|
93
136
|
|
|
94
137
|
# 有小数或全是空值,保持 float64
|
|
95
|
-
# 如果指定了decimal_places,对float类型执行round操作
|
|
96
|
-
if decimal_places is not None:
|
|
97
|
-
series = series.round(decimal_places)
|
|
98
|
-
|
|
99
138
|
return series
|
|
100
139
|
|
|
101
140
|
|
|
102
|
-
def
|
|
141
|
+
def _resolve_columns_decimal_places(
|
|
142
|
+
columns: list[str] | dict[str, int | None],
|
|
143
|
+
decimal_places: int | None,
|
|
144
|
+
) -> tuple[list[str], dict[str, int | None]]:
|
|
103
145
|
"""
|
|
104
|
-
|
|
146
|
+
解析 columns 参数,返回目标列列表和每列对应的 decimal_places 字典。
|
|
147
|
+
|
|
148
|
+
参数:
|
|
149
|
+
- columns: 列名列表或字典
|
|
150
|
+
- decimal_places: 默认的 decimal_places 值
|
|
151
|
+
|
|
152
|
+
返回:
|
|
153
|
+
- target_columns: 目标列列表
|
|
154
|
+
- column_decimal_places: 每列对应的 decimal_places 字典
|
|
155
|
+
"""
|
|
156
|
+
if isinstance(columns, list):
|
|
157
|
+
target_columns = columns
|
|
158
|
+
column_decimal_places = {col: decimal_places for col in target_columns}
|
|
159
|
+
elif isinstance(columns, dict):
|
|
160
|
+
for col, val in columns.items():
|
|
161
|
+
if val is not None and not isinstance(val, int):
|
|
162
|
+
raise TypeError(
|
|
163
|
+
f"列 '{col}' 的 decimal_places 必须是整数或 None,得到 {type(val).__name__}"
|
|
164
|
+
)
|
|
165
|
+
target_columns = list(columns.keys())
|
|
166
|
+
column_decimal_places = {col: val for col, val in columns.items()}
|
|
167
|
+
else:
|
|
168
|
+
raise TypeError(
|
|
169
|
+
f"columns 参数类型不支持,期望 List[str] 或 Dict[str, Optional[int]],"
|
|
170
|
+
f"得到 {type(columns).__name__}"
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
return target_columns, column_decimal_places
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def round_columns(
|
|
177
|
+
df: pd.DataFrame,
|
|
178
|
+
columns: list[str] | dict[str, int | None],
|
|
179
|
+
decimal_places: int = 0,
|
|
180
|
+
) -> pd.DataFrame:
|
|
181
|
+
"""
|
|
182
|
+
对DataFrame中指定列进行四舍五入操作。
|
|
183
|
+
自动支持 Decimal 类型列和 NaN 值。
|
|
105
184
|
|
|
106
185
|
参数:
|
|
107
186
|
- df (pd.DataFrame):输入的DataFrame。
|
|
108
|
-
- columns (Dict[str,
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
- decimal_places (int, optional)
|
|
187
|
+
- columns (List[str], Dict[str, Optional[int]]):
|
|
188
|
+
* 如果为List[str],则对指定列进行四舍五入
|
|
189
|
+
* 如果为Dict[str, Optional[int]],则键为列名,值为该列的decimal_places(可选)
|
|
190
|
+
- decimal_places (int, optional):保留的小数位数,默认为0。
|
|
112
191
|
|
|
113
192
|
返回:
|
|
114
|
-
- pd.DataFrame
|
|
193
|
+
- pd.DataFrame:四舍五入后的DataFrame。
|
|
115
194
|
"""
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
TYPE_MAPPING[target_type]
|
|
124
|
-
) # pyright: ignore[reportArgumentType]
|
|
125
|
-
if decimal_places is not None:
|
|
126
|
-
df[column] = df[column].round(decimal_places)
|
|
127
|
-
else:
|
|
128
|
-
df[column] = df[column].astype(
|
|
129
|
-
TYPE_MAPPING[target_type]
|
|
130
|
-
) # pyright: ignore[reportArgumentType]
|
|
131
|
-
except (ValueError, TypeError):
|
|
132
|
-
# 如果转换失败,保持原类型
|
|
133
|
-
pass
|
|
195
|
+
target_columns, column_decimal_places = _resolve_columns_decimal_places(
|
|
196
|
+
columns, decimal_places
|
|
197
|
+
)
|
|
198
|
+
for column in target_columns:
|
|
199
|
+
if column in df.columns:
|
|
200
|
+
col_dp = column_decimal_places.get(column, decimal_places)
|
|
201
|
+
df[column] = convert_numeric_series(df[column], col_dp)
|
|
134
202
|
return df
|
|
135
203
|
|
|
136
204
|
|
|
@@ -144,23 +212,22 @@ def _has_decimal(df: pd.DataFrame, column: str) -> bool:
|
|
|
144
212
|
|
|
145
213
|
def convert_decimal(
|
|
146
214
|
df: pd.DataFrame,
|
|
147
|
-
columns:
|
|
148
|
-
|
|
149
|
-
decimal_places: Optional[int] = None,
|
|
215
|
+
columns: list[str] | dict[str, int | None] | None = None,
|
|
216
|
+
decimal_places: int | None = None,
|
|
150
217
|
) -> pd.DataFrame:
|
|
151
218
|
"""
|
|
152
|
-
检测DataFrame中是否包含Decimal
|
|
219
|
+
检测DataFrame中是否包含Decimal类型的字段,如果包含则转换为数值类型。
|
|
220
|
+
自动检测:先转为float,如果没有小数则转为int,否则保持float。
|
|
221
|
+
当 decimal_places == 0 时,round后自动转为Int64类型。
|
|
153
222
|
|
|
154
223
|
参数:
|
|
155
224
|
- df (pd.DataFrame):输入的DataFrame。
|
|
156
|
-
- columns (List[str], Dict[str,
|
|
225
|
+
- columns (List[str], Dict[str, Optional[int]], or None):
|
|
157
226
|
* 如果为None,则检测所有列
|
|
158
|
-
* 如果为List[str]
|
|
159
|
-
* 如果为Dict[str,
|
|
160
|
-
-
|
|
161
|
-
|
|
162
|
-
'auto'表示自动检测:先转为float,如果没有小数则转为int。
|
|
163
|
-
- decimal_places (int, optional):当target_type为'auto'或'float'时,保留的小数位数,默认为None表示不限制
|
|
227
|
+
* 如果为List[str],则检测指定列
|
|
228
|
+
* 如果为Dict[str, Optional[int]],则键为列名,值为该列的decimal_places(可选)
|
|
229
|
+
- decimal_places (int, optional):保留的小数位数,默认为None表示不限制。
|
|
230
|
+
当decimal_places为0时,round后自动转为Int64类型。
|
|
164
231
|
|
|
165
232
|
返回:
|
|
166
233
|
- pd.DataFrame:转换后的DataFrame。
|
|
@@ -168,23 +235,11 @@ def convert_decimal(
|
|
|
168
235
|
if columns is None:
|
|
169
236
|
# 检测所有列
|
|
170
237
|
target_columns = df.columns.tolist()
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
elif isinstance(columns, dict):
|
|
177
|
-
# 使用字典指定的类型
|
|
178
|
-
target_columns = list(columns.keys())
|
|
179
|
-
column_types = columns
|
|
180
|
-
|
|
181
|
-
# 验证target_type和字典中的类型是否有效
|
|
182
|
-
valid_types = {"int", "float", "auto"}
|
|
183
|
-
for col, target_type in column_types.items():
|
|
184
|
-
if target_type not in valid_types:
|
|
185
|
-
raise ValueError(
|
|
186
|
-
f"无效的类型指定:{target_type}。支持的类型为:{valid_types}"
|
|
187
|
-
)
|
|
238
|
+
column_decimal_places = {col: decimal_places for col in target_columns}
|
|
239
|
+
else:
|
|
240
|
+
target_columns, column_decimal_places = _resolve_columns_decimal_places(
|
|
241
|
+
columns, decimal_places
|
|
242
|
+
)
|
|
188
243
|
|
|
189
244
|
for column in target_columns:
|
|
190
245
|
# 检查列是否存在且为object类型 (只有object类型列才可能包含Decimal)
|
|
@@ -192,23 +247,15 @@ def convert_decimal(
|
|
|
192
247
|
continue
|
|
193
248
|
# 检查列中是否存在 Decimal 类型值
|
|
194
249
|
if _has_decimal(df, column):
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
if col_target_type == "int":
|
|
198
|
-
df[column] = df[column].astype(int)
|
|
199
|
-
elif col_target_type == "float":
|
|
200
|
-
df[column] = df[column].astype(float)
|
|
201
|
-
if decimal_places is not None:
|
|
202
|
-
df[column] = df[column].round(decimal_places)
|
|
203
|
-
elif col_target_type == "auto":
|
|
204
|
-
df[column] = convert_numeric_series(df[column], decimal_places)
|
|
250
|
+
col_dp = column_decimal_places.get(column, decimal_places)
|
|
251
|
+
df[column] = convert_numeric_series(df[column], col_dp)
|
|
205
252
|
|
|
206
253
|
return df
|
|
207
254
|
|
|
208
255
|
|
|
209
256
|
def load_json(
|
|
210
257
|
df: pd.DataFrame,
|
|
211
|
-
columns:
|
|
258
|
+
columns: list[str],
|
|
212
259
|
empty_to_dict: bool = True,
|
|
213
260
|
) -> pd.DataFrame:
|
|
214
261
|
"""
|
|
@@ -233,7 +280,7 @@ def load_json(
|
|
|
233
280
|
|
|
234
281
|
def convert_datetime_str(
|
|
235
282
|
df: pd.DataFrame,
|
|
236
|
-
columns:
|
|
283
|
+
columns: list[str],
|
|
237
284
|
include_time: bool = True,
|
|
238
285
|
include_seconds: bool = True,
|
|
239
286
|
fail_fill: Any = "",
|
|
@@ -272,28 +319,23 @@ def convert_datetime_str(
|
|
|
272
319
|
|
|
273
320
|
|
|
274
321
|
def convert_str_datetime(
|
|
275
|
-
df: pd.DataFrame, columns:
|
|
322
|
+
df: pd.DataFrame, columns: list[str]
|
|
276
323
|
) -> pd.DataFrame:
|
|
277
324
|
"""
|
|
278
325
|
将DataFrame中指定字符串列转换为datetime类型。
|
|
279
326
|
|
|
280
327
|
参数:
|
|
281
328
|
- df (pd.DataFrame):输入的DataFrame。
|
|
282
|
-
- columns (
|
|
329
|
+
- columns (list[str]):要转换为datetime类型的列名列表。
|
|
283
330
|
|
|
284
331
|
返回:
|
|
285
332
|
- pd.DataFrame:转换后的DataFrame。
|
|
286
333
|
"""
|
|
287
|
-
def parse_datetime(value: str) -> Union[pd.Timestamp, str]:
|
|
288
|
-
try:
|
|
289
|
-
return pd.to_datetime(value)
|
|
290
|
-
except (ValueError, TypeError):
|
|
291
|
-
return value
|
|
292
|
-
|
|
293
334
|
for column in columns:
|
|
294
335
|
if column not in df.columns:
|
|
295
336
|
continue
|
|
296
337
|
|
|
297
|
-
df[column] = df[column]
|
|
338
|
+
df[column] = pd.to_datetime(df[column], errors="coerce")
|
|
339
|
+
|
|
340
|
+
return df
|
|
298
341
|
|
|
299
|
-
return df
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import pandas as pd
|
|
2
2
|
from datetime import datetime
|
|
3
3
|
from pandas.api.types import is_datetime64_any_dtype, is_timedelta64_dtype
|
|
4
|
-
from typing import
|
|
4
|
+
from typing import Any
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
# 单位 -> 对应秒数
|
|
8
8
|
_UNIT_SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400}
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
def _resolve_date(df: pd.DataFrame, date:
|
|
11
|
+
def _resolve_date(df: pd.DataFrame, date: str | datetime) -> pd.Series:
|
|
12
12
|
"""将列名或 datetime 统一转换为 pd.Series"""
|
|
13
13
|
if isinstance(date, datetime):
|
|
14
14
|
return pd.Series([date] * len(df), index=df.index)
|
|
@@ -21,7 +21,7 @@ def _resolve_date(df: pd.DataFrame, date: Union[str, datetime]) -> pd.Series:
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def fill_nat(
|
|
24
|
-
df: pd.DataFrame, columns:
|
|
24
|
+
df: pd.DataFrame, columns: list[str] | str, fill_value: Any = ""
|
|
25
25
|
) -> pd.DataFrame:
|
|
26
26
|
"""将指定列的 NaT(Datetime/Timedelta)替换为空字符串"""
|
|
27
27
|
|
|
@@ -45,11 +45,11 @@ def fill_nat(
|
|
|
45
45
|
def cal_date_diff(
|
|
46
46
|
df: pd.DataFrame,
|
|
47
47
|
target_column: str,
|
|
48
|
-
old_date:
|
|
49
|
-
new_date:
|
|
48
|
+
old_date: str | datetime,
|
|
49
|
+
new_date: str | datetime,
|
|
50
50
|
unit: str = "h",
|
|
51
51
|
decimal_places: int = 1,
|
|
52
|
-
nat:
|
|
52
|
+
nat: int | None = None
|
|
53
53
|
) -> pd.DataFrame:
|
|
54
54
|
"""
|
|
55
55
|
计算DataFrame中两列日期的时间差,并将结果填充到指定列。
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import pandas as pd
|
|
2
|
-
from pandas.api.types import is_datetime64_any_dtype, is_timedelta64_dtype
|
|
3
|
-
from typing import
|
|
2
|
+
from pandas.api.types import is_datetime64_any_dtype, is_timedelta64_dtype
|
|
3
|
+
from typing import Any
|
|
4
4
|
from .convert_utils import convert_numeric_series
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
def fill_na(
|
|
8
8
|
df: pd.DataFrame,
|
|
9
|
-
columns:
|
|
9
|
+
columns: list[str] | dict[str, Any],
|
|
10
10
|
fill_value: Any = "",
|
|
11
|
-
decimal_places:
|
|
11
|
+
decimal_places: int | None = None,
|
|
12
|
+
copy: bool = False,
|
|
12
13
|
) -> pd.DataFrame:
|
|
13
14
|
"""
|
|
14
15
|
替换DataFrame中指定列的空值为指定值。
|
|
@@ -21,13 +22,16 @@ def fill_na(
|
|
|
21
22
|
* 如果为List[str]且为空列表,则对所有列应用fill_value填充
|
|
22
23
|
- fill_value (Any, optional):当columns为列表时的默认填充值,默认为空字符串。
|
|
23
24
|
- decimal_places (int, optional):当进行数值转换时保留的小数位数,默认为None表示不限制
|
|
25
|
+
- copy (bool, optional):是否在副本上操作而非修改原 DataFrame,默认 False。
|
|
24
26
|
|
|
25
27
|
返回:
|
|
26
28
|
- pd.DataFrame:替换空值后的DataFrame。
|
|
27
29
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
+
注意:当 copy=False 时,本函数直接修改传入的 DataFrame。
|
|
31
|
+
如需保留原数据,请设置 copy=True 或在调用前使用 df.copy()。
|
|
30
32
|
"""
|
|
33
|
+
if copy:
|
|
34
|
+
df = df.copy()
|
|
31
35
|
_is_numeric_fill = isinstance(fill_value, (int, float))
|
|
32
36
|
|
|
33
37
|
if isinstance(columns, list):
|
|
@@ -64,32 +68,3 @@ def fill_na(
|
|
|
64
68
|
column_dtype = df[column].dtype
|
|
65
69
|
raise TypeError(f"处理列 '{column}' (类型: {column_dtype}) 时出错: {e}") from e
|
|
66
70
|
return df
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
def round_columns(
|
|
70
|
-
df: pd.DataFrame, columns: List[str], decimal_places: int = 0
|
|
71
|
-
) -> pd.DataFrame:
|
|
72
|
-
"""
|
|
73
|
-
对DataFrame中指定列进行四舍五入操作。
|
|
74
|
-
|
|
75
|
-
参数:
|
|
76
|
-
- df (pd.DataFrame):输入的DataFrame。
|
|
77
|
-
- columns (List[str]):要进行四舍五入的列名列表。
|
|
78
|
-
- decimal_places (int, optional):保留的小数位数,默认为0。
|
|
79
|
-
|
|
80
|
-
返回:
|
|
81
|
-
- pd.DataFrame:四舍五入后的DataFrame。
|
|
82
|
-
|
|
83
|
-
注意:
|
|
84
|
-
- 当 decimal_places 为 0 时,结果列会自动转换为 Int64(可空整数)类型。
|
|
85
|
-
- decimal.Decimal 类型在 pandas 中以 object dtype 存储,不被视为数值类型,
|
|
86
|
-
调用本函数会抛出 TypeError。需先手动将其转换为 float 类型再调用本函数。
|
|
87
|
-
"""
|
|
88
|
-
for column in columns:
|
|
89
|
-
if column in df.columns:
|
|
90
|
-
if not is_numeric_dtype(df[column]):
|
|
91
|
-
raise TypeError(f"列 '{column}' 不是数值类型,无法执行四舍五入操作")
|
|
92
|
-
df[column] = df[column].round(decimal_places)
|
|
93
|
-
if decimal_places == 0:
|
|
94
|
-
df[column] = df[column].astype('Int64')
|
|
95
|
-
return df
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
import pandas as pd
|
|
2
|
-
from typing import Union, List, Tuple
|
|
3
2
|
|
|
4
3
|
from .statistics.mask_utils import build_single_mask, build_base_mask, combine_masks
|
|
5
4
|
|
|
6
5
|
|
|
7
6
|
def pd_filter(
|
|
8
7
|
df: pd.DataFrame,
|
|
9
|
-
conditions:
|
|
8
|
+
conditions: tuple | list[tuple] | list[pd.Series],
|
|
10
9
|
logic: str = "and"
|
|
11
10
|
) -> pd.DataFrame:
|
|
12
11
|
"""
|
|
@@ -6,7 +6,7 @@ sum、mean、max、min、count、median、std、var等聚合计算。
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
import pandas as pd
|
|
9
|
-
from typing import Any,
|
|
9
|
+
from typing import Any, Literal
|
|
10
10
|
from .mask_utils import build_single_mask, build_base_mask
|
|
11
11
|
from ..convert_utils import convert_series
|
|
12
12
|
|
|
@@ -16,13 +16,13 @@ def group_agg(
|
|
|
16
16
|
group_col: str,
|
|
17
17
|
agg_col: str,
|
|
18
18
|
agg_func: str = "sum",
|
|
19
|
-
sort:
|
|
20
|
-
conditions:
|
|
19
|
+
sort: str | None = None,
|
|
20
|
+
conditions: tuple | list[tuple] | None = None,
|
|
21
21
|
logic: str = "and",
|
|
22
|
-
true_mask:
|
|
23
|
-
false_mask:
|
|
22
|
+
true_mask: pd.Series | None = None,
|
|
23
|
+
false_mask: pd.Series | None = None,
|
|
24
24
|
return_type: Literal["dict", "df", "series"] = "dict"
|
|
25
|
-
) ->
|
|
25
|
+
) -> dict[Any, int | float] | pd.DataFrame | pd.Series:
|
|
26
26
|
"""
|
|
27
27
|
按指定列分组,对另一列进行聚合统计。
|
|
28
28
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import pandas as pd
|
|
2
|
-
from
|
|
2
|
+
from collections.abc import Mapping
|
|
3
|
+
from typing import Any, Literal
|
|
3
4
|
from .mask_utils import build_single_mask, build_base_mask
|
|
4
5
|
from ..convert_utils import convert_series
|
|
5
6
|
|
|
@@ -8,14 +9,14 @@ def value_counts(
|
|
|
8
9
|
df: pd.DataFrame,
|
|
9
10
|
column: str,
|
|
10
11
|
mode: str = "count",
|
|
11
|
-
sort:
|
|
12
|
+
sort: str | None = None,
|
|
12
13
|
dropna: bool = True,
|
|
13
|
-
conditions:
|
|
14
|
+
conditions: tuple | list[tuple] | None = None,
|
|
14
15
|
logic: str = "and",
|
|
15
|
-
true_mask:
|
|
16
|
-
false_mask:
|
|
16
|
+
true_mask: pd.Series | None = None,
|
|
17
|
+
false_mask: pd.Series | None = None,
|
|
17
18
|
return_type: Literal["dict", "df", "series"] = "dict"
|
|
18
|
-
) ->
|
|
19
|
+
) -> Mapping[Any, int | float] | pd.DataFrame | pd.Series:
|
|
19
20
|
"""
|
|
20
21
|
统计DataFrame指定列中不同值的计数数据。
|
|
21
22
|
|
|
@@ -84,10 +85,10 @@ def value_counts(
|
|
|
84
85
|
|
|
85
86
|
def count(
|
|
86
87
|
df: pd.DataFrame,
|
|
87
|
-
conditions:
|
|
88
|
+
conditions: tuple | list[tuple],
|
|
88
89
|
logic: str = "and",
|
|
89
|
-
true_mask:
|
|
90
|
-
false_mask:
|
|
90
|
+
true_mask: pd.Series | None = None,
|
|
91
|
+
false_mask: pd.Series | None = None
|
|
91
92
|
) -> int:
|
|
92
93
|
"""
|
|
93
94
|
使用int sum 方法,统计DataFrame中符合条件的非空值数量。
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import pandas as pd
|
|
2
|
-
from
|
|
2
|
+
from collections.abc import Mapping
|
|
3
|
+
from typing import Any, Literal
|
|
3
4
|
from .mask_utils import build_single_mask as _original_build_single_mask, build_base_mask as _original_build_base_mask, combine_masks
|
|
4
5
|
from .count_utils import count as _original_count, value_counts as _original_value_counts
|
|
5
6
|
from .agg_utils import group_agg as _original_group_agg
|
|
@@ -32,9 +33,6 @@ class DataFrameStatistics:
|
|
|
32
33
|
self._false_mask = None
|
|
33
34
|
self._reset_base_masks()
|
|
34
35
|
|
|
35
|
-
# 方法继承
|
|
36
|
-
self.combine_masks = combine_masks
|
|
37
|
-
|
|
38
36
|
|
|
39
37
|
# 重置 true_mask 和 false_mask
|
|
40
38
|
def _reset_base_masks(self):
|
|
@@ -47,9 +45,16 @@ class DataFrameStatistics:
|
|
|
47
45
|
if self._true_mask is not None and len(self._true_mask) != len(self._df):
|
|
48
46
|
self._reset_base_masks()
|
|
49
47
|
|
|
50
|
-
def
|
|
51
|
-
|
|
52
|
-
|
|
48
|
+
def combine_masks(self, masks: list[pd.Series], logic: str = "and") -> pd.Series:
|
|
49
|
+
"""
|
|
50
|
+
合并多个布尔掩码,支持复杂的嵌套逻辑组合(委托给模块级 combine_masks 函数)
|
|
51
|
+
"""
|
|
52
|
+
return combine_masks(masks, logic)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def build_base_mask(self, conditions: list[tuple], logic: str = "and",
|
|
56
|
+
true_mask: pd.Series | None = None,
|
|
57
|
+
false_mask: pd.Series | None = None) -> pd.Series:
|
|
53
58
|
"""
|
|
54
59
|
构建基础查询条件掩码,自动使用内部掩码参数
|
|
55
60
|
|
|
@@ -71,7 +76,7 @@ class DataFrameStatistics:
|
|
|
71
76
|
return _original_build_base_mask(self._df, conditions, logic, true_mask, false_mask)
|
|
72
77
|
|
|
73
78
|
|
|
74
|
-
def build_single_mask(self, condition:
|
|
79
|
+
def build_single_mask(self, condition: tuple) -> pd.Series:
|
|
75
80
|
"""
|
|
76
81
|
构建单个掩码,用于简单条件判断,自动使用内部DataFrame
|
|
77
82
|
|
|
@@ -86,9 +91,9 @@ class DataFrameStatistics:
|
|
|
86
91
|
return _original_build_single_mask(self._df, condition)
|
|
87
92
|
|
|
88
93
|
|
|
89
|
-
def count(self, conditions:
|
|
90
|
-
true_mask:
|
|
91
|
-
false_mask:
|
|
94
|
+
def count(self, conditions: tuple | list[tuple], logic: str = "and",
|
|
95
|
+
true_mask: pd.Series | None = None,
|
|
96
|
+
false_mask: pd.Series | None = None) -> int:
|
|
92
97
|
"""
|
|
93
98
|
统计DataFrame中符合条件的非空值数量,自动使用内部掩码参数
|
|
94
99
|
|
|
@@ -114,14 +119,14 @@ class DataFrameStatistics:
|
|
|
114
119
|
self,
|
|
115
120
|
column: str,
|
|
116
121
|
mode: str = "count",
|
|
117
|
-
sort:
|
|
122
|
+
sort: str | None = None,
|
|
118
123
|
dropna: bool = True,
|
|
119
|
-
conditions:
|
|
124
|
+
conditions: tuple | list[tuple] | None = None,
|
|
120
125
|
logic: str = "and",
|
|
121
|
-
true_mask:
|
|
122
|
-
false_mask:
|
|
126
|
+
true_mask: pd.Series | None = None,
|
|
127
|
+
false_mask: pd.Series | None = None,
|
|
123
128
|
return_type: Literal["dict", "df", "series"] = "dict"
|
|
124
|
-
) ->
|
|
129
|
+
) -> Mapping[Any, int | float] | pd.DataFrame | pd.Series:
|
|
125
130
|
"""
|
|
126
131
|
统计指定列中不同值的计数数据,自动使用内部掩码参数
|
|
127
132
|
|
|
@@ -170,13 +175,13 @@ class DataFrameStatistics:
|
|
|
170
175
|
group_col: str,
|
|
171
176
|
agg_col: str,
|
|
172
177
|
agg_func: str = "sum",
|
|
173
|
-
sort:
|
|
174
|
-
conditions:
|
|
178
|
+
sort: str | None = None,
|
|
179
|
+
conditions: tuple | list[tuple] | None = None,
|
|
175
180
|
logic: str = "and",
|
|
176
|
-
true_mask:
|
|
177
|
-
false_mask:
|
|
181
|
+
true_mask: pd.Series | None = None,
|
|
182
|
+
false_mask: pd.Series | None = None,
|
|
178
183
|
return_type: Literal["dict", "df", "series"] = "dict"
|
|
179
|
-
) ->
|
|
184
|
+
) -> dict[Any, int | float] | pd.DataFrame | pd.Series:
|
|
180
185
|
"""
|
|
181
186
|
按指定列分组,对另一列进行聚合统计,自动使用内部掩码参数
|
|
182
187
|
|
|
@@ -217,7 +222,7 @@ class DataFrameStatistics:
|
|
|
217
222
|
)
|
|
218
223
|
|
|
219
224
|
|
|
220
|
-
def dataframe_info(self) ->
|
|
225
|
+
def dataframe_info(self) -> dict[str, Any]:
|
|
221
226
|
"""获取DataFrame的基本信息"""
|
|
222
227
|
return {
|
|
223
228
|
"shape": self._df.shape,
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import pandas as pd
|
|
3
|
-
from typing import List, Tuple, Optional
|
|
4
|
-
|
|
5
3
|
|
|
6
4
|
|
|
7
5
|
def _is_empty(x):
|
|
@@ -40,7 +38,7 @@ def _is_not_empty(x):
|
|
|
40
38
|
return True
|
|
41
39
|
|
|
42
40
|
|
|
43
|
-
def build_single_mask(df: pd.DataFrame, condition:
|
|
41
|
+
def build_single_mask(df: pd.DataFrame, condition: tuple) -> pd.Series:
|
|
44
42
|
"""
|
|
45
43
|
构建单个掩码,用于简单条件判断
|
|
46
44
|
|
|
@@ -121,10 +119,10 @@ def build_single_mask(df: pd.DataFrame, condition: Tuple) -> pd.Series:
|
|
|
121
119
|
|
|
122
120
|
def build_base_mask(
|
|
123
121
|
df: pd.DataFrame,
|
|
124
|
-
conditions:
|
|
122
|
+
conditions: list[tuple],
|
|
125
123
|
logic: str = "and",
|
|
126
|
-
true_mask:
|
|
127
|
-
false_mask:
|
|
124
|
+
true_mask: pd.Series | None = None,
|
|
125
|
+
false_mask: pd.Series | None = None,
|
|
128
126
|
) -> pd.Series:
|
|
129
127
|
"""
|
|
130
128
|
构建基础查询条件掩码
|
|
@@ -145,20 +143,26 @@ def build_base_mask(
|
|
|
145
143
|
说明:
|
|
146
144
|
- 如需复杂的嵌套逻辑组合,请使用 combine_masks 方法。
|
|
147
145
|
"""
|
|
146
|
+
def _and_mask(m, c):
|
|
147
|
+
return m & c
|
|
148
|
+
|
|
149
|
+
def _or_mask(m, c):
|
|
150
|
+
return m | c
|
|
151
|
+
|
|
148
152
|
if logic == "and":
|
|
149
153
|
mask = (
|
|
150
154
|
true_mask
|
|
151
155
|
if true_mask is not None
|
|
152
156
|
else pd.Series([True] * len(df), index=df.index)
|
|
153
157
|
)
|
|
154
|
-
operator_func =
|
|
158
|
+
operator_func = _and_mask
|
|
155
159
|
elif logic == "or":
|
|
156
160
|
mask = (
|
|
157
161
|
false_mask
|
|
158
162
|
if false_mask is not None
|
|
159
163
|
else pd.Series([False] * len(df), index=df.index)
|
|
160
164
|
)
|
|
161
|
-
operator_func =
|
|
165
|
+
operator_func = _or_mask
|
|
162
166
|
else:
|
|
163
167
|
raise ValueError(f"不支持的逻辑操作类型: {logic},支持 'and' 或 'or'")
|
|
164
168
|
|
|
@@ -169,7 +173,7 @@ def build_base_mask(
|
|
|
169
173
|
return mask
|
|
170
174
|
|
|
171
175
|
|
|
172
|
-
def combine_masks(masks:
|
|
176
|
+
def combine_masks(masks: list[pd.Series], logic: str = "and") -> pd.Series:
|
|
173
177
|
"""
|
|
174
178
|
合并多个布尔掩码,支持复杂的嵌套逻辑组合
|
|
175
179
|
|
|
@@ -2,9 +2,10 @@ import json
|
|
|
2
2
|
import base64
|
|
3
3
|
import hashlib
|
|
4
4
|
import requests
|
|
5
|
+
import urllib.parse
|
|
5
6
|
from curl_cffi import requests as cffi_requests
|
|
6
7
|
|
|
7
|
-
from typing import
|
|
8
|
+
from typing import Any, Literal
|
|
8
9
|
from .core import retry_function
|
|
9
10
|
from .data_models import RequestLog
|
|
10
11
|
|
|
@@ -14,7 +15,7 @@ HttpMethod = Literal["GET", "POST", "PUT", "DELETE", "OPTIONS", "HEAD", "TRACE",
|
|
|
14
15
|
|
|
15
16
|
# impersonate 名称 → 对应真实浏览器 User-Agent 映射
|
|
16
17
|
# 只维护 curl_cffi 支持的主流标识
|
|
17
|
-
_IMPERSONATE_UA_MAP:
|
|
18
|
+
_IMPERSONATE_UA_MAP: dict[str, str] = {
|
|
18
19
|
"chrome123": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
|
|
19
20
|
"chrome124": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
20
21
|
"safari15_5": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15",
|
|
@@ -54,9 +55,9 @@ def encode_basic_auth(username: str, password: str) -> str:
|
|
|
54
55
|
def curl_cffi_request(
|
|
55
56
|
method: HttpMethod,
|
|
56
57
|
url: str,
|
|
57
|
-
req_kwargs:
|
|
58
|
+
req_kwargs: dict[str, Any],
|
|
58
59
|
impersonate: str,
|
|
59
|
-
auto_retry:
|
|
60
|
+
auto_retry: dict[str, Any] | None = None,
|
|
60
61
|
) -> Any:
|
|
61
62
|
"""
|
|
62
63
|
使用 curl_cffi 发起请求的内部封装。
|
|
@@ -105,16 +106,17 @@ def curl_cffi_request(
|
|
|
105
106
|
def send_request(
|
|
106
107
|
method: HttpMethod,
|
|
107
108
|
url: str,
|
|
108
|
-
headers:
|
|
109
|
-
data:
|
|
109
|
+
headers: dict[str, str] | None = None,
|
|
110
|
+
data: Any | None = None,
|
|
111
|
+
params: dict[str, Any] | None = None,
|
|
110
112
|
return_type: str = "json",
|
|
111
113
|
timeout: int = 60,
|
|
112
|
-
auto_retry:
|
|
114
|
+
auto_retry: dict[str, Any] | None = None,
|
|
113
115
|
request_log: RequestLog = RequestLog(),
|
|
114
116
|
curl_fallback: bool = False,
|
|
115
117
|
curl_fallback_impersonate: str = _DEFAULT_IMPERSONATE,
|
|
116
118
|
stream: bool = False,
|
|
117
|
-
) ->
|
|
119
|
+
) -> dict | str | requests.Response:
|
|
118
120
|
"""
|
|
119
121
|
发送HTTP请求的通用函数
|
|
120
122
|
|
|
@@ -122,6 +124,7 @@ def send_request(
|
|
|
122
124
|
:param url: 请求URL
|
|
123
125
|
:param headers: 请求头
|
|
124
126
|
:param data: 请求数据
|
|
127
|
+
:param params: URL查询参数,dict 类型,会自动进行 URL 编码并拼接在 URL 后
|
|
125
128
|
:param return_type: 返回类型(json, text, response)
|
|
126
129
|
:param timeout: 请求超时时间
|
|
127
130
|
:param auto_retry: 自动重试配置,格式为:
|
|
@@ -139,6 +142,10 @@ def send_request(
|
|
|
139
142
|
"""
|
|
140
143
|
payload = None
|
|
141
144
|
|
|
145
|
+
# 处理 URL 查询参数
|
|
146
|
+
parameters = urllib.parse.urlencode(params) if params else ""
|
|
147
|
+
url = url + (f"?{parameters}" if parameters else "")
|
|
148
|
+
|
|
142
149
|
if data is not None:
|
|
143
150
|
if isinstance(data, dict) or isinstance(data, list):
|
|
144
151
|
payload = json.dumps(data, ensure_ascii=False)
|
|
@@ -221,7 +228,7 @@ def send_request(
|
|
|
221
228
|
# 检查URL是否有效
|
|
222
229
|
def check_url_valid(
|
|
223
230
|
url: str,
|
|
224
|
-
headers:
|
|
231
|
+
headers: dict[str, str] | None = None,
|
|
225
232
|
max_retries: int = 3,
|
|
226
233
|
curl_fallback: bool = False,
|
|
227
234
|
curl_fallback_impersonate: str = _DEFAULT_IMPERSONATE,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: funcguard
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.56
|
|
4
4
|
Summary: FuncGuard是一个Python库,提供函数执行超时控制、重试机制、HTTP请求封装和格式化打印工具。
|
|
5
5
|
Home-page: https://github.com/tinycen/funcguard
|
|
6
6
|
Author: tinycen
|
|
@@ -188,6 +188,15 @@ response = send_request(
|
|
|
188
188
|
)
|
|
189
189
|
print(response)
|
|
190
190
|
|
|
191
|
+
# 带 URL 查询参数的 GET 请求
|
|
192
|
+
response = send_request(
|
|
193
|
+
method="GET",
|
|
194
|
+
url="https://api.example.com/users",
|
|
195
|
+
params={"page": 1, "size": 20, "search": "张三"}
|
|
196
|
+
# 等价于: https://api.example.com/users?page=1&size=20&search=%E5%BC%A0%E4%B8%89
|
|
197
|
+
)
|
|
198
|
+
print(response)
|
|
199
|
+
|
|
191
200
|
# POST 请求(自动将 dict/list 转为 JSON)
|
|
192
201
|
response = send_request(
|
|
193
202
|
method="POST",
|
|
@@ -515,7 +524,7 @@ FuncGuard提供了丰富的pandas数据处理功能,包括数据填充、类
|
|
|
515
524
|
|
|
516
525
|
```python
|
|
517
526
|
import pandas as pd
|
|
518
|
-
from funcguard.pd_utils import fill_na, convert_columns,
|
|
527
|
+
from funcguard.pd_utils import fill_na, convert_columns, load_json
|
|
519
528
|
|
|
520
529
|
# 快速示例
|
|
521
530
|
df = pd.DataFrame({
|
|
@@ -622,7 +631,8 @@ print(f"当前价格: {current_price}, 变化: {price_change}") # 输出: 当
|
|
|
622
631
|
| `pd_group_agg` | 分组聚合统计 | [查看](docs/pandas/agg.md) |
|
|
623
632
|
| `pd_build_mask` / `pd_build_masks` / `pd_combine_masks` | 掩码构建 | [查看](docs/pandas/mask.md) |
|
|
624
633
|
| `DataFrameStatistics` | 统计分析 | [查看](docs/pandas/statistics.md) |
|
|
625
|
-
| `pd_cal_date_diff`
|
|
634
|
+
| `pd_cal_date_diff` | 日期计算 | [查看](docs/pandas/date.md) |
|
|
635
|
+
| `pd_round_columns` | 数值舍入 | [查看](docs/pandas/convert.md) |
|
|
626
636
|
|
|
627
637
|
### 计算工具
|
|
628
638
|
|
|
@@ -20,7 +20,7 @@ funcguard/data_models/request_models.py
|
|
|
20
20
|
funcguard/pd_utils/__init__.py
|
|
21
21
|
funcguard/pd_utils/convert_utils.py
|
|
22
22
|
funcguard/pd_utils/date_utils.py
|
|
23
|
-
funcguard/pd_utils/
|
|
23
|
+
funcguard/pd_utils/fill_utils.py
|
|
24
24
|
funcguard/pd_utils/filter.py
|
|
25
25
|
funcguard/pd_utils/json_utils/__init__.py
|
|
26
26
|
funcguard/pd_utils/json_utils/json_parser.py
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
from dataclasses import dataclass
|
|
2
|
-
|
|
3
|
-
from typing import Optional
|
|
4
|
-
|
|
5
|
-
@dataclass
|
|
6
|
-
class RequestLog:
|
|
7
|
-
save_method: Optional[bool] = True
|
|
8
|
-
save_url: Optional[bool] = True
|
|
9
|
-
save_headers: Optional[bool] = True
|
|
10
|
-
save_body: Optional[bool] = True
|
|
11
|
-
save_response: Optional[bool] = True
|
|
12
|
-
save_path: Optional[str] = ""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|