ipclick 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ipclick might be problematic. Click here for more details.
- {ipclick-0.6.0 → ipclick-0.7.0}/PKG-INFO +19 -4
- {ipclick-0.6.0 → ipclick-0.7.0}/README.md +17 -2
- {ipclick-0.6.0 → ipclick-0.7.0}/pyproject.toml +9 -2
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/__init__.py +1 -1
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/adapters/base.py +127 -2
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/adapters/browser_adapter.py +16 -3
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/adapters/browser_engines.py +9 -3
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/adapters/browser_settings.py +130 -3
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/adapters/curl_cffi_adapter.py +112 -25
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/adapters/niquests_adapter.py +57 -1
- ipclick-0.7.0/src/ipclick/async_limiter.py +238 -0
- ipclick-0.7.0/src/ipclick/async_server.py +173 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/auth.py +10 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cli/main.py +5 -2
- ipclick-0.7.0/src/ipclick/cluster/async_forwarder.py +104 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cluster/forwarder.py +10 -3
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cluster/pool.py +21 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/configs/default_config.toml +52 -3
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/limiter.py +133 -15
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/server.py +159 -1
- ipclick-0.7.0/src/ipclick/services/async_task_service.py +217 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/services/task_service.py +76 -43
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/utils/log_util.py +50 -17
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/web/editable.py +71 -2
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/web/pages.py +12 -1
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/web/templates.py +18 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/LICENSE +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/__main__.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/adapters/__init__.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/adapters/drission_adapter.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/adapters/registry.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/adapters/settings.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/aio.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cli/__init__.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cli/agent.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cli/output.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cli/skill_cmd.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cluster/__init__.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cluster/balancer.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cluster/client.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cluster/discovery.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cluster/node.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cluster/probe.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cluster/status_page.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/cluster/tokens.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/components.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/compression.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/config_loader/__init__.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/config_loader/dotenv.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/config_loader/loader.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/config_loader/placeholders.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/config_loader/writer.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/configs/__init__.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/dto/__init__.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/dto/models.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/dto/proto/__init__.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/dto/proto/generate.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/dto/proto/task.proto +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/dto/proto/task_pb2.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/dto/proto/task_pb2.pyi +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/dto/proto/task_pb2_grpc.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/dto/response.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/exceptions.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/factory.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/health.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/ports.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/py.typed +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/resume.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/sdk.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/secrets.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/services/__init__.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/skill.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/skills/ipclick/SKILL.md +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/tls.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/trace.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/utils/__init__.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/utils/config_util.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/utils/module_probe.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/utils/path_util.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/utils/secure_util.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/utils/url_util.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/web/__init__.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/web/assets.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/web/auth.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/web/curl_parser.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/web/deploy.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/web/installer.py +0 -0
- {ipclick-0.6.0 → ipclick-0.7.0}/src/ipclick/web/server.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ipclick
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: IPClick Tool
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -34,7 +34,7 @@ Requires-Dist: niquests (>=3.21.0,<4.0.0) ; extra == "niquests"
|
|
|
34
34
|
Requires-Dist: patchright (>=1.61.2,<2.0.0) ; extra == "linux"
|
|
35
35
|
Requires-Dist: patchright (>=1.61.2,<2.0.0) ; extra == "patchright"
|
|
36
36
|
Requires-Dist: playwright (>=1.60.0,<2.0.0) ; extra == "playwright"
|
|
37
|
-
Requires-Dist: protobuf (>=
|
|
37
|
+
Requires-Dist: protobuf (>=7.35.1,<8.0.0)
|
|
38
38
|
Requires-Dist: python-box[all] (>=7.4.1,<8.0.0)
|
|
39
39
|
Requires-Dist: typing-extensions (>=4.16.0,<5.0.0)
|
|
40
40
|
Requires-Dist: uuid-utils (>=0.17.0,<0.18.0)
|
|
@@ -155,6 +155,16 @@ max_concurrent_rpcs = 0 # 0 = max_workers × 8。这一项决定"排队能排
|
|
|
155
155
|
实测并发 100 时:单进程 196.6 → 331.1 QPS,四进程 → 544.6 QPS;
|
|
156
156
|
1000 并发的成功率从 25.8%(异步客户端)回到 100%。详见 CHANGELOG 的 0.6.0 一节。
|
|
157
157
|
|
|
158
|
+
0.7.0 起还可以把服务端换成协程(实验性,默认关):
|
|
159
|
+
|
|
160
|
+
```toml
|
|
161
|
+
[SERVER]
|
|
162
|
+
async_mode = true # grpc.aio + 协程,不再一请求一线程。实测端到端 1.8×
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
**协程和多进程不是二选一**:协程解决并发模型,多进程解决多核,单进程再优化
|
|
166
|
+
也只能用一个核。两者叠加才是终态。默认关的理由见 CHANGELOG 的 0.7.0 一节。
|
|
167
|
+
|
|
158
168
|
初始化配置(推荐):
|
|
159
169
|
|
|
160
170
|
```bash
|
|
@@ -293,6 +303,11 @@ IPCLICK_WEB_USER=ops IPCLICK_WEB_PASSWORD=... ipclick run -w
|
|
|
293
303
|
`[CLUSTER].allow_remote_install`:它等于允许调用方在那台机器上跑 pip,
|
|
294
304
|
是从"能代发 HTTP 请求"到"能改本机 Python 环境"的实质提权。
|
|
295
305
|
|
|
306
|
+
> **一个例外,说在前面**:Web 管理端「生成部署材料」产出的子节点 `ipclick.toml`
|
|
307
|
+
> 里,这一项是**打开**的——理由是那份配置本来就由主控生成,你已经信任它了,
|
|
308
|
+
> 否则逐台 SSH 上去装适配器很烦。不想要就把生成出来的那一行改成 `false`,
|
|
309
|
+
> 之后主控的组件页对这台会返回"未开启"。
|
|
310
|
+
|
|
296
311
|
**从 curl 导入**:浏览器 DevTools 里对着请求「复制为 cURL」,粘进输入框就能自动
|
|
297
312
|
填好 URL / 方法 / 请求头 / 请求体。认不出的参数会明确列出来,不静默丢弃。
|
|
298
313
|
|
|
@@ -310,7 +325,7 @@ IPCLICK_WEB_USER=ops IPCLICK_WEB_PASSWORD=... ipclick run -w
|
|
|
310
325
|
|
|
311
326
|
### 能改什么、不能改什么
|
|
312
327
|
|
|
313
|
-
- **能改**(写回 toml,保留注释与格式,改动前留 `.bak`):
|
|
328
|
+
- **能改**(写回 toml,保留注释与格式,改动前留 `.bak`):74 项,分 12 组——服务端、
|
|
314
329
|
日志、下载行为、重试、连接池、按 host 限流、浏览器渲染、代理、集群、链路记录、
|
|
315
330
|
Web 管理端、客户端与压缩。
|
|
316
331
|
- **不能改**:`[SECURITY]` 全部(令牌、TLS、SSRF 三个开关)、Web 自己的登录凭据、
|
|
@@ -520,7 +535,7 @@ docker run -d -p 9528:9528 --name ipclick ipclick:latest # gRPC
|
|
|
520
535
|
docker run -d -p 9528:9528 -p 9527:9527 --name ipclick ipclick:latest # 再带上 Web 管理端
|
|
521
536
|
```
|
|
522
537
|
|
|
523
|
-
镜像多阶段构建、非 root 运行。详细说明见 [`docker/构建.md`](docker
|
|
538
|
+
镜像多阶段构建、非 root 运行。详细说明见 [`docker/构建.md`](https://github.com/yuanqimanong/IPClick/blob/master/docker/%E6%9E%84%E5%BB%BA.md)。
|
|
524
539
|
|
|
525
540
|
## 🚧 已知限制
|
|
526
541
|
|
|
@@ -110,6 +110,16 @@ max_concurrent_rpcs = 0 # 0 = max_workers × 8。这一项决定"排队能排
|
|
|
110
110
|
实测并发 100 时:单进程 196.6 → 331.1 QPS,四进程 → 544.6 QPS;
|
|
111
111
|
1000 并发的成功率从 25.8%(异步客户端)回到 100%。详见 CHANGELOG 的 0.6.0 一节。
|
|
112
112
|
|
|
113
|
+
0.7.0 起还可以把服务端换成协程(实验性,默认关):
|
|
114
|
+
|
|
115
|
+
```toml
|
|
116
|
+
[SERVER]
|
|
117
|
+
async_mode = true # grpc.aio + 协程,不再一请求一线程。实测端到端 1.8×
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
**协程和多进程不是二选一**:协程解决并发模型,多进程解决多核,单进程再优化
|
|
121
|
+
也只能用一个核。两者叠加才是终态。默认关的理由见 CHANGELOG 的 0.7.0 一节。
|
|
122
|
+
|
|
113
123
|
初始化配置(推荐):
|
|
114
124
|
|
|
115
125
|
```bash
|
|
@@ -248,6 +258,11 @@ IPCLICK_WEB_USER=ops IPCLICK_WEB_PASSWORD=... ipclick run -w
|
|
|
248
258
|
`[CLUSTER].allow_remote_install`:它等于允许调用方在那台机器上跑 pip,
|
|
249
259
|
是从"能代发 HTTP 请求"到"能改本机 Python 环境"的实质提权。
|
|
250
260
|
|
|
261
|
+
> **一个例外,说在前面**:Web 管理端「生成部署材料」产出的子节点 `ipclick.toml`
|
|
262
|
+
> 里,这一项是**打开**的——理由是那份配置本来就由主控生成,你已经信任它了,
|
|
263
|
+
> 否则逐台 SSH 上去装适配器很烦。不想要就把生成出来的那一行改成 `false`,
|
|
264
|
+
> 之后主控的组件页对这台会返回"未开启"。
|
|
265
|
+
|
|
251
266
|
**从 curl 导入**:浏览器 DevTools 里对着请求「复制为 cURL」,粘进输入框就能自动
|
|
252
267
|
填好 URL / 方法 / 请求头 / 请求体。认不出的参数会明确列出来,不静默丢弃。
|
|
253
268
|
|
|
@@ -265,7 +280,7 @@ IPCLICK_WEB_USER=ops IPCLICK_WEB_PASSWORD=... ipclick run -w
|
|
|
265
280
|
|
|
266
281
|
### 能改什么、不能改什么
|
|
267
282
|
|
|
268
|
-
- **能改**(写回 toml,保留注释与格式,改动前留 `.bak`):
|
|
283
|
+
- **能改**(写回 toml,保留注释与格式,改动前留 `.bak`):74 项,分 12 组——服务端、
|
|
269
284
|
日志、下载行为、重试、连接池、按 host 限流、浏览器渲染、代理、集群、链路记录、
|
|
270
285
|
Web 管理端、客户端与压缩。
|
|
271
286
|
- **不能改**:`[SECURITY]` 全部(令牌、TLS、SSRF 三个开关)、Web 自己的登录凭据、
|
|
@@ -475,7 +490,7 @@ docker run -d -p 9528:9528 --name ipclick ipclick:latest # gRPC
|
|
|
475
490
|
docker run -d -p 9528:9528 -p 9527:9527 --name ipclick ipclick:latest # 再带上 Web 管理端
|
|
476
491
|
```
|
|
477
492
|
|
|
478
|
-
镜像多阶段构建、非 root 运行。详细说明见 [`docker/构建.md`](docker
|
|
493
|
+
镜像多阶段构建、非 root 运行。详细说明见 [`docker/构建.md`](https://github.com/yuanqimanong/IPClick/blob/master/docker/%E6%9E%84%E5%BB%BA.md)。
|
|
479
494
|
|
|
480
495
|
## 🚧 已知限制
|
|
481
496
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ipclick"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.7.0"
|
|
4
4
|
description = "IPClick Tool"
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "元气码农少女酱", email = "1276162429@qq.com" }
|
|
@@ -12,7 +12,11 @@ dependencies = [
|
|
|
12
12
|
"curl-cffi>=0.16.0,<0.17.0",
|
|
13
13
|
"grpcio>=1.83.0,<2.0.0",
|
|
14
14
|
"grpcio-health-checking>=1.83.0,<2.0.0",
|
|
15
|
-
|
|
15
|
+
# 下限必须 >= 生成代码的 gencode 版本:src/ipclick/dto/proto/task_pb2.py 里
|
|
16
|
+
# 有 ValidateProtobufRuntimeVersion(..., 7, 35, 1) 的硬校验,运行时低于它
|
|
17
|
+
# 就直接 import 失败。实际上 grpcio-health-checking 也顶着同一个下限,
|
|
18
|
+
# 所以写 6.33.2 装不出问题——但那是靠别人兜着,不是这里声明对了。
|
|
19
|
+
"protobuf>=7.35.1,<8.0.0",
|
|
16
20
|
"click>=8.4.2,<9.0.0",
|
|
17
21
|
"fake-useragent>=2.2.0,<3.0.0",
|
|
18
22
|
"loguru>=0.7.3,<0.8.0",
|
|
@@ -69,6 +73,9 @@ dev = [
|
|
|
69
73
|
"pytest>=9.1.1",
|
|
70
74
|
"pytest-cov>=7.0.0",
|
|
71
75
|
"pytest-asyncio>=1.0.0",
|
|
76
|
+
# 并行跑测试。整套串行 72 秒、-n 4 只要 25 秒(2.8×),而 CI 上测试这一步
|
|
77
|
+
# 本来就占整个 job 的 79%(177s / 223s)。
|
|
78
|
+
"pytest-xdist>=3.6.0",
|
|
72
79
|
# 测试里现生成自签名证书,用来做真实的 TLS/mTLS 握手
|
|
73
80
|
"cryptography>=42.0.0",
|
|
74
81
|
]
|
|
@@ -29,7 +29,7 @@ try:
|
|
|
29
29
|
__version__ = importlib.metadata.version("ipclick")
|
|
30
30
|
__author__ = importlib.metadata.metadata("ipclick")["Author"]
|
|
31
31
|
except importlib.metadata.PackageNotFoundError: # pragma: no cover - 仅在未安装时走到
|
|
32
|
-
__version__ = "0.
|
|
32
|
+
__version__ = "0.7.0"
|
|
33
33
|
__author__ = "元气码农少女酱"
|
|
34
34
|
|
|
35
35
|
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
|
-
|
|
2
|
+
import asyncio
|
|
3
|
+
from collections.abc import AsyncIterator, Callable, Iterator
|
|
3
4
|
from dataclasses import dataclass, field
|
|
4
5
|
import functools
|
|
5
6
|
from random import randrange, uniform
|
|
6
7
|
import re
|
|
7
8
|
import threading
|
|
8
9
|
import time
|
|
9
|
-
from typing import Any
|
|
10
|
+
from typing import Any, cast
|
|
10
11
|
|
|
11
12
|
from ipclick.adapters.settings import DEFAULT_RETRY_STATUS_CODES, AdapterSettings
|
|
12
13
|
from ipclick.dto.response import Response
|
|
@@ -186,6 +187,90 @@ def retry(
|
|
|
186
187
|
return decorator
|
|
187
188
|
|
|
188
189
|
|
|
190
|
+
def aretry(
|
|
191
|
+
max_retries_attr: str = "max_retries", retry_delay_attr: str = "retry_delay"
|
|
192
|
+
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
|
|
193
|
+
""":func:`retry` 的异步版。
|
|
194
|
+
|
|
195
|
+
单独写一份而不是复用同步那个:里面的 ``time.sleep`` 在协程里会**阻塞整个
|
|
196
|
+
事件循环**——不是拖慢这一个请求,是让同一个循环上所有在飞的请求一起停住。
|
|
197
|
+
默认退避 1+2+4 秒,一次重试就能把整个 worker 冻结七秒,而现象是"毫不相干的
|
|
198
|
+
请求也集体变慢",极难联想到是某个失败请求在退避。
|
|
199
|
+
|
|
200
|
+
行为与同步版逐条对齐:只重试真正值得重试的,ValidationError / AdapterError
|
|
201
|
+
直接抛(重试改变不了参数写错或依赖没装)。
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
|
|
205
|
+
@functools.wraps(func)
|
|
206
|
+
async def wrapper(self: Any, *args: Any, **kwargs: Any) -> Response:
|
|
207
|
+
requested_retries = kwargs.get("max_retries")
|
|
208
|
+
max_retries = max(
|
|
209
|
+
0, int(requested_retries) if requested_retries is not None else getattr(self, max_retries_attr, 3)
|
|
210
|
+
)
|
|
211
|
+
requested_delay = kwargs.get("retry_delay")
|
|
212
|
+
base_delay = _coerce_delay(
|
|
213
|
+
requested_delay if requested_delay is not None else getattr(self, retry_delay_attr, 1.0),
|
|
214
|
+
default=1.0,
|
|
215
|
+
)
|
|
216
|
+
url = args[0] if args else kwargs.get("url", "unknown")
|
|
217
|
+
allowed = kwargs.get("allowed_status_codes") or None
|
|
218
|
+
|
|
219
|
+
settings: AdapterSettings | None = getattr(self, "settings", None)
|
|
220
|
+
retry_codes = settings.retry_codes if settings else DEFAULT_RETRY_STATUS_CODES
|
|
221
|
+
exponent = settings.backoff_exponent if settings else 2.0
|
|
222
|
+
max_backoff = settings.max_backoff if settings else MAX_RETRY_DELAY
|
|
223
|
+
|
|
224
|
+
last_exception: Exception | None = None
|
|
225
|
+
for attempt in range(max_retries + 1):
|
|
226
|
+
start_time = time.monotonic()
|
|
227
|
+
try:
|
|
228
|
+
result = await func(self, *args, **kwargs)
|
|
229
|
+
if hasattr(result, "elapsed_ms") and result.elapsed_ms == 0:
|
|
230
|
+
result.elapsed_ms = int((time.monotonic() - start_time) * 1000)
|
|
231
|
+
if hasattr(result, "attempts"):
|
|
232
|
+
result.attempts = attempt + 1
|
|
233
|
+
|
|
234
|
+
status = getattr(result, "status_code", None)
|
|
235
|
+
if (
|
|
236
|
+
attempt < max_retries
|
|
237
|
+
and isinstance(status, int)
|
|
238
|
+
and status in retry_codes
|
|
239
|
+
and not (allowed and status in allowed)
|
|
240
|
+
):
|
|
241
|
+
sleep_time = _backoff(attempt, base_delay, exponent, max_backoff)
|
|
242
|
+
get_recorder().record_retry(getattr(self, "adapter_name", "unknown"), "status_code")
|
|
243
|
+
log.warning(
|
|
244
|
+
f"Download {url} returned {status}, retrying {attempt + 1}/{max_retries} "
|
|
245
|
+
f"in {sleep_time:.1f}s..."
|
|
246
|
+
)
|
|
247
|
+
await asyncio.sleep(sleep_time)
|
|
248
|
+
continue
|
|
249
|
+
return result
|
|
250
|
+
|
|
251
|
+
except (ValidationError, AdapterError):
|
|
252
|
+
raise
|
|
253
|
+
except Exception as e:
|
|
254
|
+
last_exception = e
|
|
255
|
+
if attempt >= max_retries:
|
|
256
|
+
return Response.error_response(url, e, attempts=attempt + 1)
|
|
257
|
+
sleep_time = _backoff(attempt, base_delay, exponent, max_backoff)
|
|
258
|
+
get_recorder().record_retry(getattr(self, "adapter_name", "unknown"), "exception")
|
|
259
|
+
log.warning(
|
|
260
|
+
f"Download {url} failed, retrying {attempt + 1}/{max_retries} "
|
|
261
|
+
f"in {sleep_time:.1f}s... Error: {e}"
|
|
262
|
+
)
|
|
263
|
+
await asyncio.sleep(sleep_time)
|
|
264
|
+
|
|
265
|
+
return Response.error_response(
|
|
266
|
+
url, last_exception or Exception("Max retries exceeded"), attempts=max_retries + 1
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
return wrapper
|
|
270
|
+
|
|
271
|
+
return decorator
|
|
272
|
+
|
|
273
|
+
|
|
189
274
|
def _backoff(
|
|
190
275
|
attempt: int,
|
|
191
276
|
base_delay: float,
|
|
@@ -371,6 +456,46 @@ class DownloaderAdapter(ABC):
|
|
|
371
456
|
"""
|
|
372
457
|
raise NotImplementedError
|
|
373
458
|
|
|
459
|
+
#: 这个适配器有没有自己的异步实现。
|
|
460
|
+
#:
|
|
461
|
+
#: 服务端的异步模式据此决定走哪条路:True 就 await adownload(),
|
|
462
|
+
#: False 就把同步的 download() 丢进线程池。**默认 False**——这一条是
|
|
463
|
+
#: 整套异步化能做成"加法"而不是破坏性变更的关键:项目支持注册自定义
|
|
464
|
+
#: 适配器(见 README),它们只实现了同步 download(),不该因为服务端换了
|
|
465
|
+
#: 并发模型就集体失效。
|
|
466
|
+
supports_async: bool = False
|
|
467
|
+
|
|
468
|
+
async def adownload(self, url: str, **kwargs: Any) -> Response:
|
|
469
|
+
"""异步执行一次请求。
|
|
470
|
+
|
|
471
|
+
默认实现把同步的 :meth:`download` 丢进线程池——**语义完全一致,
|
|
472
|
+
只是拿不到协程的好处**(那个线程仍然是稀缺资源)。真正想要收益的
|
|
473
|
+
适配器覆写此方法并把 ``supports_async`` 置 True。
|
|
474
|
+
|
|
475
|
+
为什么不把 download 直接改成 async:那会打断每一个第三方适配器,
|
|
476
|
+
而它们看不到任何提示——基类方法签名变了,子类的同步实现会被当成
|
|
477
|
+
协程去 await,报出来的错误("object Response can't be used in
|
|
478
|
+
'await' expression")和真正的原因("你的适配器该改成 async 了")
|
|
479
|
+
之间没有任何字面联系。
|
|
480
|
+
"""
|
|
481
|
+
loop = asyncio.get_running_loop()
|
|
482
|
+
return await loop.run_in_executor(None, functools.partial(self.download, url, **kwargs))
|
|
483
|
+
|
|
484
|
+
async def adownload_stream(self, url: str, **kwargs: Any) -> "AsyncIterator[StreamEvent]":
|
|
485
|
+
"""异步流式。默认把同步迭代器逐个搬到线程池里取。
|
|
486
|
+
|
|
487
|
+
逐个搬而不是一次性取完:流式的意义就在于响应体不整个进内存,
|
|
488
|
+
先 list() 再 yield 等于把这个性质丢掉。
|
|
489
|
+
"""
|
|
490
|
+
loop = asyncio.get_running_loop()
|
|
491
|
+
iterator = await loop.run_in_executor(None, functools.partial(self.download_stream, url, **kwargs))
|
|
492
|
+
sentinel = object()
|
|
493
|
+
while True:
|
|
494
|
+
item = await loop.run_in_executor(None, next, iterator, sentinel)
|
|
495
|
+
if item is sentinel:
|
|
496
|
+
return
|
|
497
|
+
yield cast(StreamEvent, item)
|
|
498
|
+
|
|
374
499
|
def download_stream(
|
|
375
500
|
self,
|
|
376
501
|
url: str,
|
|
@@ -47,7 +47,12 @@ from ipclick.adapters.base import (
|
|
|
47
47
|
raise_if_script_error,
|
|
48
48
|
retry,
|
|
49
49
|
)
|
|
50
|
-
from ipclick.adapters.browser_settings import
|
|
50
|
+
from ipclick.adapters.browser_settings import (
|
|
51
|
+
BLOCKABLE_RESOURCES,
|
|
52
|
+
WAIT_UNTIL_CHOICES,
|
|
53
|
+
BrowserSettings,
|
|
54
|
+
resolve_max_pages,
|
|
55
|
+
)
|
|
51
56
|
from ipclick.adapters.settings import AdapterSettings
|
|
52
57
|
from ipclick.dto.response import Response
|
|
53
58
|
from ipclick.exceptions import AdapterError, ValidationError
|
|
@@ -83,6 +88,8 @@ class _BrowserWorker:
|
|
|
83
88
|
def __init__(self, settings: BrowserSettings, engine: str):
|
|
84
89
|
self._settings: BrowserSettings = settings
|
|
85
90
|
self._engine: str = engine
|
|
91
|
+
#: 上一次算出来的页面上限,只用于变化时打一条日志,避免每次重建都刷屏
|
|
92
|
+
self._resolved_pages: int = 0
|
|
86
93
|
self._loop: asyncio.AbstractEventLoop | None = None
|
|
87
94
|
self._thread: threading.Thread | None = None
|
|
88
95
|
self._start_lock: threading.Lock = threading.Lock()
|
|
@@ -146,8 +153,14 @@ class _BrowserWorker:
|
|
|
146
153
|
launched = await browser_engines.launch(self._engine, self._settings)
|
|
147
154
|
self._playwright = launched.driver
|
|
148
155
|
self._browser = launched.browser
|
|
149
|
-
#
|
|
150
|
-
|
|
156
|
+
# 信号量跟着浏览器一起重建:旧的那个可能还扣着已经消失的页面的额度。
|
|
157
|
+
# max_pages = 0 时按本机内存推导(浏览器页面是内存瓶颈,不是 CPU),
|
|
158
|
+
# 每次重建都重算一次——机器上别的进程占了多少内存是会变的。
|
|
159
|
+
limit = resolve_max_pages(self._settings.max_pages, self._engine)
|
|
160
|
+
if limit != self._resolved_pages:
|
|
161
|
+
self._resolved_pages = limit
|
|
162
|
+
log.info(f"{self._engine} 页面并发上限:{limit}(max_pages={self._settings.max_pages or 'auto'})")
|
|
163
|
+
self._semaphore = asyncio.Semaphore(limit)
|
|
151
164
|
return self._browser
|
|
152
165
|
|
|
153
166
|
@staticmethod
|
|
@@ -34,7 +34,7 @@ import sys
|
|
|
34
34
|
import threading
|
|
35
35
|
from typing import Any, final
|
|
36
36
|
|
|
37
|
-
from ipclick.adapters.browser_settings import BrowserSettings
|
|
37
|
+
from ipclick.adapters.browser_settings import BrowserSettings, describe_max_pages
|
|
38
38
|
from ipclick.exceptions import AdapterError, ConfigError
|
|
39
39
|
from ipclick.utils import module_probe
|
|
40
40
|
from ipclick.utils.log_util import log
|
|
@@ -484,7 +484,10 @@ async def _launch_playwright_like(api: Any, engine: str, settings: BrowserSettin
|
|
|
484
484
|
await driver.stop()
|
|
485
485
|
raise AdapterError(f"浏览器启动失败({engine} / {settings.kind}):{e}。{INSTALL_HINTS[engine]}") from e
|
|
486
486
|
|
|
487
|
-
log.info(
|
|
487
|
+
log.info(
|
|
488
|
+
f"{engine} 浏览器已启动:{settings.kind}, headless={settings.headless}, "
|
|
489
|
+
f"页面上限 {describe_max_pages(settings.max_pages, engine)}"
|
|
490
|
+
)
|
|
488
491
|
return LaunchedBrowser(driver=driver, browser=browser)
|
|
489
492
|
|
|
490
493
|
|
|
@@ -551,7 +554,10 @@ async def _launch_camoufox(settings: BrowserSettings) -> LaunchedBrowser:
|
|
|
551
554
|
await driver.stop()
|
|
552
555
|
raise AdapterError(f"Camoufox 启动失败:{e}。{INSTALL_HINTS['camoufox']}") from e
|
|
553
556
|
|
|
554
|
-
log.info(
|
|
557
|
+
log.info(
|
|
558
|
+
f"camoufox 浏览器已启动:headless={settings.headless}, "
|
|
559
|
+
f"页面上限 {describe_max_pages(settings.max_pages, 'camoufox')}"
|
|
560
|
+
)
|
|
555
561
|
return LaunchedBrowser(driver=driver, browser=browser)
|
|
556
562
|
|
|
557
563
|
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
from dataclasses import dataclass, field
|
|
12
|
+
import os
|
|
12
13
|
from typing import Any
|
|
13
14
|
|
|
14
15
|
|
|
@@ -85,7 +86,10 @@ class BrowserSettings:
|
|
|
85
86
|
#: 默认拦截的资源类型。图片 / 字体 / 视频对取 HTML 没用,拦掉能省大量时间和带宽。
|
|
86
87
|
block_resources: tuple[str, ...] = ("image", "media", "font")
|
|
87
88
|
|
|
88
|
-
#:
|
|
89
|
+
#: 同时打开的页面上限。``0`` = 按本机内存自动推导(见 :func:`resolve_max_pages`)。
|
|
90
|
+
#:
|
|
91
|
+
#: 默认仍是 4 而不是 0——自动推导会因机器而异,静默改变既有部署的行为不合适。
|
|
92
|
+
#: 想让它跟着机器走就显式写 0。
|
|
89
93
|
max_pages: int = 4
|
|
90
94
|
|
|
91
95
|
#: 是否允许请求携带 automation_script(在页面里执行任意 JS)。
|
|
@@ -161,7 +165,11 @@ class BrowserSettings:
|
|
|
161
165
|
script_timeout=_as_float(timeout.get("script_exec"), defaults.script_timeout),
|
|
162
166
|
wait_until=wait_until,
|
|
163
167
|
block_resources=block_resources,
|
|
164
|
-
|
|
168
|
+
# minimum=0 是关键:0 不是"非法的小值",而是"按可用内存自动推导"这个
|
|
169
|
+
# 特性的开关(见 resolve_max_pages)。用默认的 minimum=1 会把它判成
|
|
170
|
+
# 低于下限、静默回落到静态默认值 4 —— 于是 max_pages = 0 和不写这一行
|
|
171
|
+
# 完全等价,整个自动推导从配置文件根本走不到。
|
|
172
|
+
max_pages=_as_int(config.get("max_pages"), defaults.max_pages, minimum=0),
|
|
165
173
|
allow_scripts=bool(config.get("allow_scripts", defaults.allow_scripts)),
|
|
166
174
|
proxy_gateway=gateway,
|
|
167
175
|
proxy_bypass=_as_str_tuple(proxy.get("bypass_list")),
|
|
@@ -171,4 +179,123 @@ class BrowserSettings:
|
|
|
171
179
|
)
|
|
172
180
|
|
|
173
181
|
|
|
174
|
-
__all__ = [
|
|
182
|
+
__all__ = [
|
|
183
|
+
"BLOCKABLE_RESOURCES",
|
|
184
|
+
"BROWSER_KINDS",
|
|
185
|
+
"WAIT_UNTIL_CHOICES",
|
|
186
|
+
"BrowserSettings",
|
|
187
|
+
"describe_max_pages",
|
|
188
|
+
"resolve_max_pages",
|
|
189
|
+
]
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
#: 每个并发页面的内存预算(MB),按引擎区分。
|
|
193
|
+
#:
|
|
194
|
+
#: 这些不是精确值,是**保守的量级**:真实占用随页面复杂度浮动很大(一个静态
|
|
195
|
+
#: 页几十 MB,一个重 JS 的应用能到几百)。取保守值的理由是失败代价不对称——
|
|
196
|
+
#: 少开一个页面只是慢一点,多开一个把机器推进 swap 就是请求从几秒变几分钟、
|
|
197
|
+
#: 看起来像卡死,而且很难从现象联想到是并发开太大。
|
|
198
|
+
#:
|
|
199
|
+
#: camoufox 明显更贵:它是 Firefox 加一整套反检测扩展,单个 context 的开销
|
|
200
|
+
#: 高于 Chromium 系。配置注释里"内存紧张的机器(≤4GB)建议设成 1~2"说的
|
|
201
|
+
#: 就是它。
|
|
202
|
+
ENGINE_PAGE_BUDGET_MB: dict[str, int] = {
|
|
203
|
+
"camoufox": 400,
|
|
204
|
+
"playwright": 250,
|
|
205
|
+
"patchright": 250,
|
|
206
|
+
"drissionpage": 250,
|
|
207
|
+
}
|
|
208
|
+
DEFAULT_PAGE_BUDGET_MB = 300
|
|
209
|
+
|
|
210
|
+
#: 自动推导时给系统留的余量(MB)。把内存吃到一滴不剩比少开几个页面糟得多。
|
|
211
|
+
MEMORY_HEADROOM_MB = 1024
|
|
212
|
+
|
|
213
|
+
#: 自动推导的硬上限。再多的页面收益也会被目标站点和网络吃掉,
|
|
214
|
+
#: 而每个页面都是一份实打实的内存。
|
|
215
|
+
MAX_AUTO_PAGES = 16
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def available_memory_mb() -> int | None:
|
|
219
|
+
"""本机可用内存(MB)。读不出来返回 None,由调用方回落到静态默认值。
|
|
220
|
+
|
|
221
|
+
优先读 cgroup 的限额而不是宿主机总量:容器里 ``/proc/meminfo`` 报的是
|
|
222
|
+
**宿主机**的内存,照它推导会在一个 512MB 的容器里开出十几个页面,
|
|
223
|
+
然后被 OOM killer 杀掉——而现象只是"容器莫名其妙重启"。
|
|
224
|
+
"""
|
|
225
|
+
import os
|
|
226
|
+
from pathlib import Path as _Path
|
|
227
|
+
|
|
228
|
+
limits: list[int] = []
|
|
229
|
+
|
|
230
|
+
# cgroup v2 → v1,取能读到的那个
|
|
231
|
+
for path, current in (
|
|
232
|
+
("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory.current"),
|
|
233
|
+
("/sys/fs/cgroup/memory/memory.limit_in_bytes", "/sys/fs/cgroup/memory/memory.usage_in_bytes"),
|
|
234
|
+
):
|
|
235
|
+
try:
|
|
236
|
+
raw = _Path(path).read_text(encoding="utf-8").strip()
|
|
237
|
+
if raw in ("max", ""):
|
|
238
|
+
continue
|
|
239
|
+
total = int(raw)
|
|
240
|
+
# 没设限时内核会报一个天文数字,那等于"不受限"
|
|
241
|
+
if total <= 0 or total > (1 << 50):
|
|
242
|
+
continue
|
|
243
|
+
used = int(_Path(current).read_text(encoding="utf-8").strip() or 0)
|
|
244
|
+
limits.append(max(0, total - used) // (1024 * 1024))
|
|
245
|
+
break
|
|
246
|
+
except (OSError, ValueError):
|
|
247
|
+
continue
|
|
248
|
+
|
|
249
|
+
try:
|
|
250
|
+
meminfo = _Path("/proc/meminfo").read_text(encoding="utf-8")
|
|
251
|
+
for line in meminfo.splitlines():
|
|
252
|
+
if line.startswith("MemAvailable:"):
|
|
253
|
+
limits.append(int(line.split()[1]) // 1024)
|
|
254
|
+
break
|
|
255
|
+
except (OSError, ValueError, IndexError):
|
|
256
|
+
pass
|
|
257
|
+
|
|
258
|
+
if not limits:
|
|
259
|
+
try:
|
|
260
|
+
page_size = os.sysconf("SC_PAGE_SIZE")
|
|
261
|
+
limits.append(os.sysconf("SC_AVPHYS_PAGES") * page_size // (1024 * 1024))
|
|
262
|
+
except (ValueError, OSError, AttributeError):
|
|
263
|
+
return None
|
|
264
|
+
|
|
265
|
+
return min(limits) if limits else None
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def resolve_max_pages(configured: int, engine: str) -> int:
|
|
269
|
+
"""算出真正生效的页面并发上限。
|
|
270
|
+
|
|
271
|
+
``configured > 0`` 就照用——显式配置永远优先,自动推导不该覆盖人的决定。
|
|
272
|
+
|
|
273
|
+
``configured == 0`` 时按**内存**推导,而不是 CPU 核数。这一点容易搞反:
|
|
274
|
+
浏览器页面是内存瓶颈不是 CPU 瓶颈,按核数算的话一台 16 核 4GB 的机器会
|
|
275
|
+
开出 16 个 camoufox 页面(约 6.4GB),直接换页。CPU 核数只做上限——
|
|
276
|
+
页面渲染确实要 CPU,开得比核数多也没意义。
|
|
277
|
+
"""
|
|
278
|
+
if configured > 0:
|
|
279
|
+
return configured
|
|
280
|
+
|
|
281
|
+
budget = ENGINE_PAGE_BUDGET_MB.get(engine.lower(), DEFAULT_PAGE_BUDGET_MB)
|
|
282
|
+
available = available_memory_mb()
|
|
283
|
+
if available is None:
|
|
284
|
+
# 读不出内存就退回静态默认值,别自作聪明
|
|
285
|
+
return BrowserSettings.max_pages
|
|
286
|
+
|
|
287
|
+
usable = max(0, available - MEMORY_HEADROOM_MB)
|
|
288
|
+
by_memory = usable // budget
|
|
289
|
+
by_cpu = os.cpu_count() or 1
|
|
290
|
+
return max(1, min(by_memory, by_cpu, MAX_AUTO_PAGES))
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def describe_max_pages(configured: int, engine: str) -> str:
|
|
294
|
+
"""给人看的页面上限。
|
|
295
|
+
|
|
296
|
+
只显示配置的原始值会误导:配了 ``0``(自动推导)的人看到"页面上限 0",
|
|
297
|
+
合理的推断是"并发被关掉了"或"这个特性没生效",而实际生效的是推导出来的
|
|
298
|
+
那个数。状态页、CLI、启动日志三处都曾这么显示。
|
|
299
|
+
"""
|
|
300
|
+
resolved = resolve_max_pages(configured, engine)
|
|
301
|
+
return str(resolved) if configured > 0 else f"{resolved}(auto)"
|