why-tools 0.2.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- venv/bin/activate_this.py +36 -0
- venv314/bin/activate_this.py +38 -0
- why_tools-0.2.26.dist-info/METADATA +558 -0
- why_tools-0.2.26.dist-info/RECORD +51 -0
- why_tools-0.2.26.dist-info/WHEEL +5 -0
- why_tools-0.2.26.dist-info/top_level.txt +3 -0
- ytools/VERSION +1 -0
- ytools/__init__.py +35 -0
- ytools/alert/__init__.py +11 -0
- ytools/alert/wechat.py +132 -0
- ytools/arq/__init__.py +11 -0
- ytools/arq/client/__init__.py +11 -0
- ytools/arq/client/agent.py +160 -0
- ytools/arq/client/base.py +129 -0
- ytools/arq/client/client.py +99 -0
- ytools/arq/setting.py +23 -0
- ytools/arq/task/__init__.py +11 -0
- ytools/arq/task/task.py +114 -0
- ytools/auto_driver/__init__.py +10 -0
- ytools/auto_driver/dp/__init__.py +10 -0
- ytools/auto_driver/dp/route.py +439 -0
- ytools/auto_driver/dp/route_by_fetch.py +403 -0
- ytools/auto_driver/dp/rpa.py +270 -0
- ytools/auto_driver/rpa_base.py +194 -0
- ytools/auto_driver/track.py +356 -0
- ytools/error.py +28 -0
- ytools/log.py +14 -0
- ytools/network/__init__.py +14 -0
- ytools/network/header.py +248 -0
- ytools/network/request.py +295 -0
- ytools/network/response.py +406 -0
- ytools/others/__init__.py +11 -0
- ytools/others/no_eol.py +213 -0
- ytools/utils/__init__.py +10 -0
- ytools/utils/cache.py +87 -0
- ytools/utils/cache_utils.py +210 -0
- ytools/utils/counter.py +76 -0
- ytools/utils/date.py +102 -0
- ytools/utils/encrypt.py +91 -0
- ytools/utils/extractors.py +438 -0
- ytools/utils/file.py +24 -0
- ytools/utils/host_ip.py +28 -0
- ytools/utils/magic.py +609 -0
- ytools/utils/magic_for_class.py +134 -0
- ytools/utils/nodes.py +158 -0
- ytools/utils/package.py +630 -0
- ytools/utils/quiter.py +71 -0
- ytools/utils/sign.py +20 -0
- ytools/utils/track.py +383 -0
- ytools/utils/variable.py +668 -0
- ytools/version.py +97 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Activate virtualenv for current interpreter:
|
|
3
|
+
|
|
4
|
+
Use exec(open(this_file).read(), {'__file__': this_file}).
|
|
5
|
+
|
|
6
|
+
This can be used when you must use an existing Python interpreter, not the virtualenv bin/python.
|
|
7
|
+
""" # noqa: D415
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import site
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
abs_file = os.path.abspath(__file__)
|
|
16
|
+
except NameError as exc:
|
|
17
|
+
msg = "You must use exec(open(this_file).read(), {'__file__': this_file}))"
|
|
18
|
+
raise AssertionError(msg) from exc
|
|
19
|
+
|
|
20
|
+
bin_dir = os.path.dirname(abs_file)
|
|
21
|
+
base = bin_dir[: -len("bin") - 1] # strip away the bin part from the __file__, plus the path separator
|
|
22
|
+
|
|
23
|
+
# prepend bin to PATH (this file is inside the bin directory)
|
|
24
|
+
os.environ["PATH"] = os.pathsep.join([bin_dir, *os.environ.get("PATH", "").split(os.pathsep)])
|
|
25
|
+
os.environ["VIRTUAL_ENV"] = base # virtual env is right above bin directory
|
|
26
|
+
os.environ["VIRTUAL_ENV_PROMPT"] = "" or os.path.basename(base) # noqa: SIM222
|
|
27
|
+
|
|
28
|
+
# add the virtual environments libraries to the host python import mechanism
|
|
29
|
+
prev_length = len(sys.path)
|
|
30
|
+
for lib in "../lib/python3.12/site-packages".split(os.pathsep):
|
|
31
|
+
path = os.path.realpath(os.path.join(bin_dir, lib))
|
|
32
|
+
site.addsitedir(path.decode("utf-8") if "" else path)
|
|
33
|
+
sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
|
|
34
|
+
|
|
35
|
+
sys.real_prefix = sys.prefix
|
|
36
|
+
sys.prefix = base
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Activate virtualenv for current interpreter:
|
|
3
|
+
|
|
4
|
+
import runpy
|
|
5
|
+
runpy.run_path(this_file)
|
|
6
|
+
|
|
7
|
+
This can be used when you must use an existing Python interpreter, not the virtualenv bin/python.
|
|
8
|
+
""" # noqa: D415
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
import site
|
|
14
|
+
import sys
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
abs_file = os.path.abspath(__file__)
|
|
18
|
+
except NameError as exc:
|
|
19
|
+
msg = "You must use import runpy; runpy.run_path(this_file)"
|
|
20
|
+
raise AssertionError(msg) from exc
|
|
21
|
+
|
|
22
|
+
bin_dir = os.path.dirname(abs_file)
|
|
23
|
+
base = bin_dir[: -len('bin') - 1] # strip away the bin part from the __file__, plus the path separator
|
|
24
|
+
|
|
25
|
+
# prepend bin to PATH (this file is inside the bin directory)
|
|
26
|
+
os.environ["PATH"] = os.pathsep.join([bin_dir, *os.environ.get("PATH", "").split(os.pathsep)])
|
|
27
|
+
os.environ["VIRTUAL_ENV"] = base # virtual env is right above bin directory
|
|
28
|
+
os.environ["VIRTUAL_ENV_PROMPT"] = '' or os.path.basename(base)
|
|
29
|
+
|
|
30
|
+
# add the virtual environments libraries to the host python import mechanism
|
|
31
|
+
prev_length = len(sys.path)
|
|
32
|
+
for lib in '../lib/python3.14/site-packages'.split(os.pathsep):
|
|
33
|
+
path = os.path.realpath(os.path.join(bin_dir, lib))
|
|
34
|
+
site.addsitedir(path.decode("utf-8") if '' else path)
|
|
35
|
+
sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
|
|
36
|
+
|
|
37
|
+
sys.real_prefix = sys.prefix
|
|
38
|
+
sys.prefix = base
|
|
@@ -0,0 +1,558 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: why-tools
|
|
3
|
+
Version: 0.2.26
|
|
4
|
+
Summary: 一个偏脚手架 / 内部工具箱风格的 Python 工具集合
|
|
5
|
+
Author-email: yintian <yintian710@gmail.com>
|
|
6
|
+
Maintainer-email: yintian <yintian710@gmail.com>
|
|
7
|
+
Project-URL: Homepage, https://github.com/yintian710/whytools
|
|
8
|
+
Project-URL: Repository, https://github.com/yintian710/whytools
|
|
9
|
+
Keywords: ytools,why-tools
|
|
10
|
+
Requires-Python: >=3.8
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
Requires-Dist: better_exceptions==0.3.3
|
|
13
|
+
Requires-Dist: importlib-metadata==7.1.0
|
|
14
|
+
Requires-Dist: loguru==0.7.3
|
|
15
|
+
Requires-Dist: packaging>=24.0
|
|
16
|
+
|
|
17
|
+
# ytools
|
|
18
|
+
|
|
19
|
+
一个偏“脚手架 / 内部工具箱”风格的 Python 工具集合,面向脚本开发、自动化、爬虫/RPA、异步任务分发和一些常见的胶水代码场景。
|
|
20
|
+
|
|
21
|
+
这个项目不是单一职责库,而是把一批作者常用能力放进了同一个包里,按需导入即可。
|
|
22
|
+
|
|
23
|
+
> 代码中的发布名是 `why-tools`,实际导入名是 `ytools`
|
|
24
|
+
>
|
|
25
|
+
> 代码要求 Python `>=3.8`
|
|
26
|
+
|
|
27
|
+
## 这个项目有什么功能
|
|
28
|
+
|
|
29
|
+
- 动态依赖与反射工具:按字符串加载对象、自动补齐函数参数、统一调用同步/异步函数
|
|
30
|
+
- 变量容器:提供全局级、线程级、协程级变量空间,并支持类型约束、过期次数和过期时间
|
|
31
|
+
- HTTP 数据模型:封装 `Request`、`Response`、`Header`、`Cookies`,便于保存、转换和解析请求/响应数据
|
|
32
|
+
- 数据提取:支持 `xpath`、`json/jmespath`、`jsonpath`、`regex` 多种提取方式
|
|
33
|
+
- Redis 异步任务队列:提供生产者 `Client`、消费者 `Agent`、任务对象 `Task`
|
|
34
|
+
- 浏览器自动化辅助:基于 `DrissionPage` 封装页面流转、元素查找、点击输入、网络拦截和响应改写
|
|
35
|
+
- 轨迹录制与回放:录制 PC / Android 的滑轨轨迹,并按距离生成可复用轨迹
|
|
36
|
+
- 通知能力:发送企业微信机器人消息
|
|
37
|
+
- 其他常用工具:日志、时间、加密、退出回调、版本号处理、链表/树节点、计数器、本机 IP 等
|
|
38
|
+
|
|
39
|
+
## 项目结构
|
|
40
|
+
|
|
41
|
+
| 模块 | 作用 | 入口对象 |
|
|
42
|
+
| --- | --- | --- |
|
|
43
|
+
| `ytools.utils.magic` | 动态加载对象、自动安装依赖、智能调用函数 | `require` `load_object` `prepare` `result` |
|
|
44
|
+
| `ytools.utils.variable` | 全局 / 线程 / 协程变量容器 | `VariableG` `VariableT` `VariableC` |
|
|
45
|
+
| `ytools.network` | 请求/响应数据模型与解析 | `Request` `Response` `Header` `Cookies` |
|
|
46
|
+
| `ytools.utils.extractors` | xpath / json / jsonpath / regex 提取器 | `Rule` `Group` `XpathExtractor` 等 |
|
|
47
|
+
| `ytools.arq` | 基于 Redis 的轻量任务队列 | `Client` `Agent` `Task` |
|
|
48
|
+
| `ytools.auto_driver.dp` | DrissionPage 自动化与请求拦截 | `DpRpaBase` `Route` |
|
|
49
|
+
| `ytools.alert.wechat` | 企业微信机器人通知 | `send_wechat` `send_msg` |
|
|
50
|
+
| `ytools.utils.track` / `ytools.auto_driver.track` | 轨迹录制、轨迹生成 | `listener` `get_track` |
|
|
51
|
+
|
|
52
|
+
## 安装
|
|
53
|
+
|
|
54
|
+
### 本地开发安装
|
|
55
|
+
|
|
56
|
+
推荐直接用 `uv` 安装开发环境:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
uv sync
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
如果你仍然使用 `pip`,也可以继续执行:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
python -m pip install -r requirements.txt
|
|
66
|
+
python -m pip install -e .
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
### 关于可选依赖
|
|
70
|
+
|
|
71
|
+
这个项目大量使用 `ytools.utils.magic.require()` 按需加载依赖。也就是说:
|
|
72
|
+
|
|
73
|
+
- 某些模块第一次导入时,会自动检查依赖是否已安装
|
|
74
|
+
- 如果机器上有 `uv`,默认优先走 `uv add`
|
|
75
|
+
- 否则会退回 `pip install`
|
|
76
|
+
|
|
77
|
+
常见的可选依赖包括:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
python -m pip install packaging redis arrow jmespath jsonpath lxml w3lib requests httpx pynput DrissionPage
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
如果你不希望运行时自动修改环境,建议提前手动装好这些包。
|
|
84
|
+
|
|
85
|
+
### 安装注意事项
|
|
86
|
+
|
|
87
|
+
- 发布推荐执行 `python publish.py`
|
|
88
|
+
- 构建底层走 `uv build` / `uv publish`
|
|
89
|
+
- `setup.py` 仅保留给旧工具链做兼容入口,不再承担发布逻辑
|
|
90
|
+
|
|
91
|
+
## 快速开始
|
|
92
|
+
|
|
93
|
+
### 1) 常用胶水工具
|
|
94
|
+
|
|
95
|
+
`ytools.utils.magic` 是整个项目里复用最广的基础模块。
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from ytools.utils.magic import load_object, result, json_or_eval
|
|
99
|
+
|
|
100
|
+
pow_func = load_object("math.pow")
|
|
101
|
+
print(pow_func(2, 10)) # 1024.0
|
|
102
|
+
|
|
103
|
+
# 自动补参数、统一执行
|
|
104
|
+
value = result("math.pow", args=[2, 8])
|
|
105
|
+
print(value) # 256.0
|
|
106
|
+
|
|
107
|
+
print(json_or_eval('{"name": "ytools", "ok": true}'))
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
适合这些场景:
|
|
111
|
+
|
|
112
|
+
- 配置里写函数路径,运行时再加载
|
|
113
|
+
- 同步 / 异步函数统一调度
|
|
114
|
+
- 给回调函数自动补 `kwargs`
|
|
115
|
+
- 快速把 JSON / 类 JSON 字符串还原成对象
|
|
116
|
+
|
|
117
|
+
### 2) 变量空间
|
|
118
|
+
|
|
119
|
+
根包里预留了 3 类变量容器:
|
|
120
|
+
|
|
121
|
+
- `G`:全局共享
|
|
122
|
+
- `T`:线程隔离
|
|
123
|
+
- `C`:协程隔离
|
|
124
|
+
|
|
125
|
+
使用前先初始化:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
import ytools
|
|
129
|
+
|
|
130
|
+
ytools.init_var("G")
|
|
131
|
+
ytools.init_var("T")
|
|
132
|
+
ytools.init_var("C")
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
#### 基础用法
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
import ytools
|
|
139
|
+
|
|
140
|
+
ytools.init_var("G")
|
|
141
|
+
|
|
142
|
+
ytools.G.user = {"name": "zz"}
|
|
143
|
+
ytools.G["count"] = 1
|
|
144
|
+
|
|
145
|
+
print(ytools.G.user) # {'name': 'zz'}
|
|
146
|
+
print(ytools.G["count"]) # 1
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
#### 类型化变量
|
|
150
|
+
|
|
151
|
+
`VariableG` 支持把变量和类型绑定在一起。
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
import ytools
|
|
155
|
+
|
|
156
|
+
ytools.init_var("G")
|
|
157
|
+
|
|
158
|
+
ytools.G["int:retry"] = 1
|
|
159
|
+
ytools.G["int:retry"] += 1
|
|
160
|
+
print(ytools.G["int:retry"]) # 2
|
|
161
|
+
|
|
162
|
+
# _list.todos 会自动按 list 类型创建默认值
|
|
163
|
+
ytools.G._list.todos.append("write readme")
|
|
164
|
+
print(ytools.G._list.todos) # ['write readme']
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
#### 过期控制
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
import ytools
|
|
171
|
+
|
|
172
|
+
ytools.init_var("G")
|
|
173
|
+
ytools.G.set("token", "abc", max_count=3, expire_ts=60)
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
其中:
|
|
177
|
+
|
|
178
|
+
- `max_count`:最多可使用次数
|
|
179
|
+
- `expire_ts`:相对秒数
|
|
180
|
+
|
|
181
|
+
### 3) Request / Response 模型
|
|
182
|
+
|
|
183
|
+
`ytools.network` 提供的是“请求/响应对象模型”,不是直接发请求的下载器。
|
|
184
|
+
|
|
185
|
+
你可以把它当成:
|
|
186
|
+
|
|
187
|
+
- curl 和结构化请求之间的转换工具
|
|
188
|
+
- 下载器结果的统一封装对象
|
|
189
|
+
- 后续提取、调试、保存的中间层
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
from ytools.network import Request, Response
|
|
193
|
+
|
|
194
|
+
req = Request(
|
|
195
|
+
url="https://example.com/api",
|
|
196
|
+
params={"page": 1},
|
|
197
|
+
method="POST",
|
|
198
|
+
json_data={"name": "demo"},
|
|
199
|
+
headers={"Content-Type": "application/json"},
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
print(req.real_url)
|
|
203
|
+
print(req.curl)
|
|
204
|
+
|
|
205
|
+
res = Response(
|
|
206
|
+
content='{"items": [{"name": "A"}]}',
|
|
207
|
+
headers={"Content-Type": "application/json"},
|
|
208
|
+
url=req.real_url,
|
|
209
|
+
request=req,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
print(res.json())
|
|
213
|
+
print(res.get("items[0].name"))
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
#### 从 curl 导入
|
|
217
|
+
|
|
218
|
+
```python
|
|
219
|
+
from ytools.network import Request
|
|
220
|
+
|
|
221
|
+
req = Request.from_curl(
|
|
222
|
+
"curl 'https://example.com?a=1' -H 'User-Agent: demo' --data 'x=1'"
|
|
223
|
+
)
|
|
224
|
+
print(req.method)
|
|
225
|
+
print(req.headers)
|
|
226
|
+
print(req.curl)
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
#### 响应提取
|
|
230
|
+
|
|
231
|
+
```python
|
|
232
|
+
from ytools.network import Response
|
|
233
|
+
|
|
234
|
+
res = Response(
|
|
235
|
+
content="""
|
|
236
|
+
<html><body><h1>Hello</h1></body></html>
|
|
237
|
+
""",
|
|
238
|
+
headers={"Content-Type": "text/html; charset=utf-8"},
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
print(res.xpath_first("//h1/text()")) # Hello
|
|
242
|
+
print(res.re_first(r"<h1>(.*?)</h1>")) # Hello
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
常用方法:
|
|
246
|
+
|
|
247
|
+
- `res.json()`:反序列化 JSON / JSONP
|
|
248
|
+
- `res.get(rule)`:用 `jmespath` 取值
|
|
249
|
+
- `res.jsonpath(expr)`:用 `jsonpath` 取值
|
|
250
|
+
- `res.xpath(expr)`:用 xpath 取值
|
|
251
|
+
- `res.re(regex)`:用正则提取
|
|
252
|
+
|
|
253
|
+
## 数据提取规则
|
|
254
|
+
|
|
255
|
+
`ytools.utils.extractors` 提供一层更灵活的规则表达。
|
|
256
|
+
|
|
257
|
+
```python
|
|
258
|
+
from ytools.utils.extractors import Rule
|
|
259
|
+
from ytools.network import Response
|
|
260
|
+
|
|
261
|
+
res = Response(content='{"name": "demo", "items": [{"id": 1}]}')
|
|
262
|
+
|
|
263
|
+
rules = {
|
|
264
|
+
"name": Rule("name"),
|
|
265
|
+
"first_id": Rule("items[0].id"),
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
print(res.extract("JSON", rules))
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
你可以用它做:
|
|
272
|
+
|
|
273
|
+
- 多字段批量抽取
|
|
274
|
+
- 字段兜底
|
|
275
|
+
- 条件命中
|
|
276
|
+
- 前置 / 后置处理
|
|
277
|
+
- 将嵌套结构展开成行数据
|
|
278
|
+
|
|
279
|
+
## Redis 异步任务队列
|
|
280
|
+
|
|
281
|
+
`ytools.arq` 是项目内置的一套轻量任务队列,不是外部那个同名框架。
|
|
282
|
+
|
|
283
|
+
### 核心角色
|
|
284
|
+
|
|
285
|
+
- `Client`:投递任务、等待结果
|
|
286
|
+
- `Agent`:消费任务、执行函数、回写结果
|
|
287
|
+
- `Task`:任务体,负责编码、解码、状态管理
|
|
288
|
+
|
|
289
|
+
### 生产者
|
|
290
|
+
|
|
291
|
+
```python
|
|
292
|
+
import asyncio
|
|
293
|
+
|
|
294
|
+
from ytools.arq import setting
|
|
295
|
+
from ytools.arq.client.client import Client
|
|
296
|
+
|
|
297
|
+
setting.OBJ_DATA = True
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
async def main():
|
|
301
|
+
client = Client(redis={"host": "127.0.0.1", "port": 6379, "db": 0})
|
|
302
|
+
|
|
303
|
+
task = await client.put(
|
|
304
|
+
{
|
|
305
|
+
"func": "math.pow",
|
|
306
|
+
"args": [2, 10],
|
|
307
|
+
},
|
|
308
|
+
auto_ensure=True,
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
raw = await client.get_result(task, timeout=5)
|
|
312
|
+
print(task.decode_data(raw))
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
asyncio.run(main())
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
### 消费者
|
|
319
|
+
|
|
320
|
+
```python
|
|
321
|
+
import asyncio
|
|
322
|
+
|
|
323
|
+
from ytools.arq.client.agent import Agent
|
|
324
|
+
from ytools.arq.task.task import Task
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
async def worker(task: Task):
|
|
328
|
+
payload = task.decode_data() if isinstance(task.data, (str, bytes)) else task.data
|
|
329
|
+
return {"ok": True, "payload": payload}
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
async def main():
|
|
333
|
+
agent = Agent(
|
|
334
|
+
worker=worker,
|
|
335
|
+
redis={"host": "127.0.0.1", "port": 6379, "db": 0},
|
|
336
|
+
max_concurrency=10,
|
|
337
|
+
)
|
|
338
|
+
await agent.run()
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
asyncio.run(main())
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
### 队列特性
|
|
345
|
+
|
|
346
|
+
- 任务按 `score` 进入 Redis 有序集合
|
|
347
|
+
- 可选加密:通过 `ytools.arq.setting.ENCRYPT` 控制
|
|
348
|
+
- 结果通过 Redis pub/sub 回传
|
|
349
|
+
- 内置心跳:客户端和消费者会定期上报自身信息
|
|
350
|
+
- 支持任务状态读写:`set_status` / `get_status` / `del_status`
|
|
351
|
+
|
|
352
|
+
## 浏览器自动化与请求拦截
|
|
353
|
+
|
|
354
|
+
这一部分依赖 `DrissionPage`,主要分两类:
|
|
355
|
+
|
|
356
|
+
- `DpRpaBase`:页面流转、元素查找、点击输入、标签页管理
|
|
357
|
+
- `Route` / `route_by_fetch.Route`:在浏览器里拦截请求或响应,并动态改写内容
|
|
358
|
+
|
|
359
|
+
### RPA 基类
|
|
360
|
+
|
|
361
|
+
`DpRpaBase` 在 `RPAControl` 基础上抽象了“当前页面在哪、下一步去哪里、处于某个页面时做什么”这套流程控制方式。
|
|
362
|
+
|
|
363
|
+
你可以继承它,实现:
|
|
364
|
+
|
|
365
|
+
- `where()`:当前页面状态识别
|
|
366
|
+
- `where_in_xxx()`:当处于某页面时执行什么动作
|
|
367
|
+
- `a_to_b()`:从页面 A 导航到页面 B
|
|
368
|
+
|
|
369
|
+
同时直接复用这些现成能力:
|
|
370
|
+
|
|
371
|
+
- `find()`:按定位表达式找元素
|
|
372
|
+
- `click()`:点击元素
|
|
373
|
+
- `input()`:输入文本
|
|
374
|
+
- `goto()`:打开页面
|
|
375
|
+
- `re_tab()` / `close_other_tab()`:整理标签页
|
|
376
|
+
- `run_slider()`:按轨迹拖动滑块
|
|
377
|
+
|
|
378
|
+
### 拦截响应并改写
|
|
379
|
+
|
|
380
|
+
```python
|
|
381
|
+
from DrissionPage._configs.chromium_options import ChromiumOptions
|
|
382
|
+
from DrissionPage._pages.chromium_page import ChromiumPage
|
|
383
|
+
|
|
384
|
+
from ytools.auto_driver.dp.route_by_fetch import Route, RouteResponse
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def mock_res(response: RouteResponse):
|
|
388
|
+
response.text = response.text.replace("old-version", "new-version")
|
|
389
|
+
response.full_headers = {
|
|
390
|
+
**response.headers,
|
|
391
|
+
"content-length": str(len(response.full_content)),
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
opt = ChromiumOptions()
|
|
396
|
+
browser = ChromiumPage(opt)
|
|
397
|
+
|
|
398
|
+
Route.start_by_driver(browser.driver)
|
|
399
|
+
Route.on("response", "main.js", mock_res)
|
|
400
|
+
|
|
401
|
+
browser.get("https://example.com")
|
|
402
|
+
```
|
|
403
|
+
|
|
404
|
+
适合这些场景:
|
|
405
|
+
|
|
406
|
+
- 调试前端接口
|
|
407
|
+
- 本地 mock 某个 JS / JSON 响应
|
|
408
|
+
- 修改请求头、请求体
|
|
409
|
+
- 观察关键接口流量
|
|
410
|
+
|
|
411
|
+
## 轨迹录制与滑块轨迹
|
|
412
|
+
|
|
413
|
+
`ytools.utils.track` 和 `ytools.auto_driver.track` 用来录制轨迹、保存轨迹、按目标距离生成一段“看起来更像人工”的滑动轨迹。
|
|
414
|
+
|
|
415
|
+
典型用途:
|
|
416
|
+
|
|
417
|
+
- 滑块验证码
|
|
418
|
+
- 鼠标轨迹采样
|
|
419
|
+
- Android 触摸轨迹采样
|
|
420
|
+
|
|
421
|
+
示例:
|
|
422
|
+
|
|
423
|
+
```python
|
|
424
|
+
from ytools.auto_driver.track import get_track
|
|
425
|
+
|
|
426
|
+
track = get_track(distance=180, file_name="fast.json")
|
|
427
|
+
print(track[:5])
|
|
428
|
+
```
|
|
429
|
+
|
|
430
|
+
## 企业微信通知
|
|
431
|
+
|
|
432
|
+
```python
|
|
433
|
+
from ytools.alert.wechat import send_msg
|
|
434
|
+
|
|
435
|
+
send_msg(
|
|
436
|
+
{"service": "demo", "status": "ok"},
|
|
437
|
+
to=["https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=YOUR_KEY"],
|
|
438
|
+
title="部署通知",
|
|
439
|
+
)
|
|
440
|
+
```
|
|
441
|
+
|
|
442
|
+
支持:
|
|
443
|
+
|
|
444
|
+
- 纯文本消息 `send_wechat(..., msg_type="text")`
|
|
445
|
+
- Markdown 消息 `send_msg(...)`
|
|
446
|
+
- 异步发送 `async_send_wechat()` / `async_send_msg()`
|
|
447
|
+
|
|
448
|
+
## 其他常用工具
|
|
449
|
+
|
|
450
|
+
### 时间
|
|
451
|
+
|
|
452
|
+
```python
|
|
453
|
+
from ytools.utils.date import Arrow
|
|
454
|
+
|
|
455
|
+
now = Arrow.now()
|
|
456
|
+
print(now.format())
|
|
457
|
+
print(now.start("day"))
|
|
458
|
+
print(now.end("day"))
|
|
459
|
+
print(now.ts(13))
|
|
460
|
+
```
|
|
461
|
+
|
|
462
|
+
### 加密与编码
|
|
463
|
+
|
|
464
|
+
```python
|
|
465
|
+
from ytools.utils.encrypt import md5, to_b64, from_b64
|
|
466
|
+
|
|
467
|
+
print(md5({"a": 1}))
|
|
468
|
+
token = to_b64("hello")
|
|
469
|
+
print(from_b64(token))
|
|
470
|
+
```
|
|
471
|
+
|
|
472
|
+
### 退出回调
|
|
473
|
+
|
|
474
|
+
```python
|
|
475
|
+
from ytools.utils.quiter import at_exit
|
|
476
|
+
|
|
477
|
+
at_exit(lambda: print("program exit"))
|
|
478
|
+
```
|
|
479
|
+
|
|
480
|
+
### 计数器
|
|
481
|
+
|
|
482
|
+
```python
|
|
483
|
+
from ytools.utils.counter import FastWriteCounter
|
|
484
|
+
|
|
485
|
+
counter = FastWriteCounter()
|
|
486
|
+
counter.increment()
|
|
487
|
+
counter.increment()
|
|
488
|
+
print(counter.value)
|
|
489
|
+
```
|
|
490
|
+
|
|
491
|
+
## 建议的使用方式
|
|
492
|
+
|
|
493
|
+
如果你第一次接触这个项目,建议按下面顺序理解:
|
|
494
|
+
|
|
495
|
+
1. 先看 `test/test_var.py`,了解变量容器怎么用
|
|
496
|
+
2. 再看 `test/arq_put.py`、`test/arq_get.py`,了解任务队列
|
|
497
|
+
3. 如果你做浏览器自动化,再看 `test/route_test.py`
|
|
498
|
+
4. 最后按需读 `ytools/utils/magic.py`,它是整个项目的基础胶水层
|
|
499
|
+
|
|
500
|
+
## 当前更适合什么场景
|
|
501
|
+
|
|
502
|
+
这个项目更适合:
|
|
503
|
+
|
|
504
|
+
- 内部工具脚本
|
|
505
|
+
- 自动化任务
|
|
506
|
+
- 爬虫 / 抓包辅助
|
|
507
|
+
- Redis 分发型任务系统
|
|
508
|
+
- 基于浏览器驱动的 RPA 场景
|
|
509
|
+
|
|
510
|
+
如果你需要的是:
|
|
511
|
+
|
|
512
|
+
- 成熟统一的 Web 框架
|
|
513
|
+
- 标准化 HTTP 客户端
|
|
514
|
+
- 强约束、强稳定性的公共 SDK
|
|
515
|
+
|
|
516
|
+
那这个项目更像一个“工具箱”,而不是完整框架。
|
|
517
|
+
|
|
518
|
+
## 已知注意点
|
|
519
|
+
|
|
520
|
+
- 很多模块是“按需依赖”,首次使用可能触发自动安装
|
|
521
|
+
- `network` 模块是请求/响应模型,不负责真正发 HTTP 请求
|
|
522
|
+
- `auto_driver` 强依赖浏览器环境和 `DrissionPage`
|
|
523
|
+
- `cache_utils.py` / `cache.py` 里有一部分缓存装饰器接口仍偏实验状态,更适合按源码理解后再使用
|
|
524
|
+
- `setup.py` 无参数执行会进入上传逻辑,日常开发不要直接跑
|
|
525
|
+
|
|
526
|
+
## 版本
|
|
527
|
+
|
|
528
|
+
当前仓库内版本文件为:
|
|
529
|
+
|
|
530
|
+
```text
|
|
531
|
+
0.2.24
|
|
532
|
+
```
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
## 发布
|
|
537
|
+
|
|
538
|
+
推荐使用项目根目录下的 `publish.py`:
|
|
539
|
+
|
|
540
|
+
```bash
|
|
541
|
+
python publish.py
|
|
542
|
+
```
|
|
543
|
+
|
|
544
|
+
常见参数:
|
|
545
|
+
|
|
546
|
+
```bash
|
|
547
|
+
python publish.py -t minor # 升中版本后发布
|
|
548
|
+
python publish.py -t pre@b # 升 beta 版本后发布
|
|
549
|
+
python publish.py -ts 0 # 跳过测试直接发布
|
|
550
|
+
```
|
|
551
|
+
|
|
552
|
+
脚本会按顺序执行:
|
|
553
|
+
|
|
554
|
+
- `uv run pytest test/ --verbose`
|
|
555
|
+
- 更新 `ytools/VERSION` 和 `pyproject.toml` 中的版本号
|
|
556
|
+
- `uv build`
|
|
557
|
+
- `uv publish`
|
|
558
|
+
- 提交 git,并在正式版本时打 tag
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
venv/bin/activate_this.py,sha256=fvPzMbbVJWNOv7QziL9ekbJ118JbxvK4JxXtdylsMzI,1337
|
|
2
|
+
venv314/bin/activate_this.py,sha256=U_0VHkjLZlNY4u_M2oAszWECrZqE8acTs9hnplfCnyg,1287
|
|
3
|
+
ytools/VERSION,sha256=EaFOmgN2oW0cjJqPU9zcQ_RBWJ7HiYB-GhEt9JScNjA,7
|
|
4
|
+
ytools/__init__.py,sha256=cErDRz6b9s5RreSAjOHoSu47G5CDmnlWSYJnhhrj5AE,749
|
|
5
|
+
ytools/error.py,sha256=1_W5yn3naglH88GHRShkmiWr5LMTvKV8bUSGAdbcsx8,617
|
|
6
|
+
ytools/log.py,sha256=MdImcAG7uTUGuLBQKv86obkZus_qtukwQAB5tYDGRDU,208
|
|
7
|
+
ytools/version.py,sha256=H-BMKpeShtrBpx3WHG0L1oATUGo6hZzryaE05aNRkyw,2809
|
|
8
|
+
ytools/alert/__init__.py,sha256=5TNviDql5RUfkJcur13rT3SmowxX_pWTIcmb5MkcVGU,171
|
|
9
|
+
ytools/alert/wechat.py,sha256=XmK_0WMrzrlcqYViBvLd0ucTJeRwfJBMM5KQDRgB2LI,5035
|
|
10
|
+
ytools/arq/__init__.py,sha256=Q4UVQsDDZVTFwC33GeDnryxWAsHlrElygNrzIw4k5HI,173
|
|
11
|
+
ytools/arq/setting.py,sha256=uHIVVd1cFtyYva9C7jS66c9Tjo7QYuFEiNWZZdJBptQ,468
|
|
12
|
+
ytools/arq/client/__init__.py,sha256=Q4UVQsDDZVTFwC33GeDnryxWAsHlrElygNrzIw4k5HI,173
|
|
13
|
+
ytools/arq/client/agent.py,sha256=C7jPzFNzlUgSwZYSYZIihC0gCQDkefmXlFo2CAq2S50,5574
|
|
14
|
+
ytools/arq/client/base.py,sha256=EOw9Skegom9zk43WnRXtCN70pxZ9YbcluKuCyJ2hC3o,4024
|
|
15
|
+
ytools/arq/client/client.py,sha256=znKf8iAULtJDz3s7MLO2qGfDG3LV-8ZDWCOA7CCLch8,3667
|
|
16
|
+
ytools/arq/task/__init__.py,sha256=JGw8Bk1PfHUQgrQAAkV6-nMexbLRin6aKZERQ-oqNVU,173
|
|
17
|
+
ytools/arq/task/task.py,sha256=5cWa122VrlED-za07SJ8pqzYC8cf-G8vQWCRVNcf3ok,3701
|
|
18
|
+
ytools/auto_driver/__init__.py,sha256=0QhrWTRwAsh95VKeZIqsHGgJc4udQJ8J_n-ujXFCxts,158
|
|
19
|
+
ytools/auto_driver/rpa_base.py,sha256=olyng3HujwEtUn_NhWZLuzUNa1zz-lfl7PpurpMlYqg,6021
|
|
20
|
+
ytools/auto_driver/track.py,sha256=QdtpA7UMov_0axtHJcxQkKfjndvjpvPA-RKl01fqVOg,10475
|
|
21
|
+
ytools/auto_driver/dp/__init__.py,sha256=awc2TeF2tl5S4Y23i6sDGKkht-_VASa8Jehj7ybh2mI,158
|
|
22
|
+
ytools/auto_driver/dp/route.py,sha256=-7CHpVSnRJEkp7S12CJpuCLqMZZ0K6Q15m1iJAaY37M,14995
|
|
23
|
+
ytools/auto_driver/dp/route_by_fetch.py,sha256=lEEVEAhfZ5unKTiYuoSrbws_wR4v8pWt9zFByWA9rBY,12822
|
|
24
|
+
ytools/auto_driver/dp/rpa.py,sha256=Sbj85sIrUeu_Qszz8S_DC-pc0OEA5UF2w9dDaFrrWX4,9800
|
|
25
|
+
ytools/network/__init__.py,sha256=Ke3KIB1akX2xDwsevCypcGKKx-bIcZPE8lIxobMJb6Q,251
|
|
26
|
+
ytools/network/header.py,sha256=I47emgok6qZgJ2-OadXZlHY0lPDnkCmczIlB27A9EmU,8064
|
|
27
|
+
ytools/network/request.py,sha256=3K5c3ONeQZWYd2tjyypZlYKUrIK8FE-HZZ7Z0RHB9DM,11777
|
|
28
|
+
ytools/network/response.py,sha256=pCjK2K2VMH_vNMQtOo58sGoo6ORgScxkKcZy0O2BPIk,10644
|
|
29
|
+
ytools/others/__init__.py,sha256=2NYrJ9n7ZBB4eYCP1CZL9nWWK632Zg6-bJtLMtZxu0I,168
|
|
30
|
+
ytools/others/no_eol.py,sha256=GPp3KOTathni0BXXZvGBUkSVNV73gXFqKqOHqxV5iM0,6391
|
|
31
|
+
ytools/utils/__init__.py,sha256=xRjC5wTEJBt6BzIkYXORfh-0xvcZwlJQvas6V1sTvYk,158
|
|
32
|
+
ytools/utils/cache.py,sha256=NlbS5WE8eNJKzu5re0dWhZv3SKxVEYEyoL2Sy1sdDtM,2782
|
|
33
|
+
ytools/utils/cache_utils.py,sha256=We8lGvc8K-7wMM-ofuo0v9DoSOr_opIo_ZCxZKcNH-Y,5718
|
|
34
|
+
ytools/utils/counter.py,sha256=ET083Eb2VAa9qCEHVEI6UwhWKrLTefw1Cxbo_ZzEw6o,1554
|
|
35
|
+
ytools/utils/date.py,sha256=qaP-LP0hL0Qf5qXYCIOvMSsl8_RZUxqLg59hcphlIdk,2002
|
|
36
|
+
ytools/utils/encrypt.py,sha256=C6r4k_UkzHqwagHsZPDLOF82DgVJDBPg3uck7mDtHpc,2676
|
|
37
|
+
ytools/utils/extractors.py,sha256=HO528qlf99Wi9RKvj0VgRJILiC6f3iEoYU0r8NnkX6M,12316
|
|
38
|
+
ytools/utils/file.py,sha256=s0TOSwU1S9I78LfLGoohFy4nuvkqhLg2KD41og1qq0Y,566
|
|
39
|
+
ytools/utils/host_ip.py,sha256=cpNFTNPLdfR_ZUSrxONSXRi8XkILMdMAGxjQ4o2ZmHU,654
|
|
40
|
+
ytools/utils/magic.py,sha256=73JZmJ5rftqA-w-48ugt5-_bVqMD2N8i_1L58bWs2ck,18302
|
|
41
|
+
ytools/utils/magic_for_class.py,sha256=MSMQ-Qg8pMat9__dUcr6Cs6GkARzO8zXuxgL3DMAffY,4644
|
|
42
|
+
ytools/utils/nodes.py,sha256=Tz-5g0Yoi2lxPq98P4g-435HKf5FVd90PRjDuR5swzY,4581
|
|
43
|
+
ytools/utils/package.py,sha256=d4yZg8UkcETF5uDJHhdblrf21xF5nm0MnuQVdVzSSVE,17551
|
|
44
|
+
ytools/utils/quiter.py,sha256=l5Et9gVyUO4mrMgB3uYBu688FePfrhquIlUus3ZHCEQ,1598
|
|
45
|
+
ytools/utils/sign.py,sha256=XBu1urPHKynYmppLAphILkizbjzpHXnkIaL9P6EeczE,289
|
|
46
|
+
ytools/utils/track.py,sha256=CN7KyoOW72hQKgFZxGQGSvg6jhwVXjDva3LF9dOcRhE,11827
|
|
47
|
+
ytools/utils/variable.py,sha256=fNBOBA9Uo3Zx1griL-aGRP5FxJLUlA1tfrBKxU9zrSY,24563
|
|
48
|
+
why_tools-0.2.26.dist-info/METADATA,sha256=A75CFJ1HAYkoAXHpuZC5fwdlu15ue-rjtBiK4SAH7iM,13915
|
|
49
|
+
why_tools-0.2.26.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
50
|
+
why_tools-0.2.26.dist-info/top_level.txt,sha256=gqfoLKuPASStzLapuWtDQaVe9kT5mUvHNy5YEYkNPjY,20
|
|
51
|
+
why_tools-0.2.26.dist-info/RECORD,,
|
ytools/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.2.26
|