pyproxytools 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyproxytools-0.3.2/LICENSE +21 -0
- pyproxytools-0.3.2/PKG-INFO +130 -0
- pyproxytools-0.3.2/README.md +98 -0
- pyproxytools-0.3.2/benchmark/benchmark.py +165 -0
- pyproxytools-0.3.2/benchmark/utils/__init__.py +0 -0
- pyproxytools-0.3.2/benchmark/utils/html.py +179 -0
- pyproxytools-0.3.2/benchmark/utils/req.py +43 -0
- pyproxytools-0.3.2/pyproject.toml +43 -0
- pyproxytools-0.3.2/pyproxy/__init__.py +13 -0
- pyproxytools-0.3.2/pyproxy/handlers/__init__.py +0 -0
- pyproxytools-0.3.2/pyproxy/handlers/client.py +126 -0
- pyproxytools-0.3.2/pyproxy/handlers/http.py +197 -0
- pyproxytools-0.3.2/pyproxy/handlers/https.py +308 -0
- pyproxytools-0.3.2/pyproxy/modules/__init__.py +0 -0
- pyproxytools-0.3.2/pyproxy/modules/cancel_inspect.py +83 -0
- pyproxytools-0.3.2/pyproxy/modules/custom_header.py +78 -0
- pyproxytools-0.3.2/pyproxy/modules/filter.py +151 -0
- pyproxytools-0.3.2/pyproxy/modules/shortcuts.py +85 -0
- pyproxytools-0.3.2/pyproxy/monitoring/__init__.py +0 -0
- pyproxytools-0.3.2/pyproxy/monitoring/web.py +279 -0
- pyproxytools-0.3.2/pyproxy/pyproxy.py +107 -0
- pyproxytools-0.3.2/pyproxy/server.py +334 -0
- pyproxytools-0.3.2/pyproxy/utils/__init__.py +0 -0
- pyproxytools-0.3.2/pyproxy/utils/args.py +176 -0
- pyproxytools-0.3.2/pyproxy/utils/config.py +110 -0
- pyproxytools-0.3.2/pyproxy/utils/crypto.py +52 -0
- pyproxytools-0.3.2/pyproxy/utils/http_req.py +53 -0
- pyproxytools-0.3.2/pyproxy/utils/logger.py +46 -0
- pyproxytools-0.3.2/pyproxy/utils/version.py +0 -0
- pyproxytools-0.3.2/pyproxytools.egg-info/PKG-INFO +130 -0
- pyproxytools-0.3.2/pyproxytools.egg-info/SOURCES.txt +45 -0
- pyproxytools-0.3.2/pyproxytools.egg-info/dependency_links.txt +1 -0
- pyproxytools-0.3.2/pyproxytools.egg-info/entry_points.txt +2 -0
- pyproxytools-0.3.2/pyproxytools.egg-info/requires.txt +6 -0
- pyproxytools-0.3.2/pyproxytools.egg-info/top_level.txt +9 -0
- pyproxytools-0.3.2/requirements.txt +6 -0
- pyproxytools-0.3.2/setup.cfg +4 -0
- pyproxytools-0.3.2/setup.py +12 -0
- pyproxytools-0.3.2/tests/modules/__init__.py +0 -0
- pyproxytools-0.3.2/tests/modules/test_cancel_inspect.py +67 -0
- pyproxytools-0.3.2/tests/modules/test_custom_header.py +70 -0
- pyproxytools-0.3.2/tests/modules/test_filter.py +185 -0
- pyproxytools-0.3.2/tests/modules/test_shortcuts.py +119 -0
- pyproxytools-0.3.2/tests/utils/__init__.py +0 -0
- pyproxytools-0.3.2/tests/utils/test_crypto.py +110 -0
- pyproxytools-0.3.2/tests/utils/test_http_req.py +69 -0
- pyproxytools-0.3.2/tests/utils/test_logger.py +68 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 6C656C65
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pyproxytools
|
|
3
|
+
Version: 0.3.2
|
|
4
|
+
Summary: Lightweight and fast python web proxy
|
|
5
|
+
Author: 6C656C65
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Documentation, https://github.com/6C656C65/pyproxy/wiki
|
|
8
|
+
Project-URL: Issue tracker, https://github.com/6C656C65/pyproxy/issues
|
|
9
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Natural Language :: English
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Internet
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
20
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
21
|
+
Classifier: Topic :: Utilities
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: rich-argparse>=1.7.0
|
|
26
|
+
Requires-Dist: pyOpenSSL>=25.0.0
|
|
27
|
+
Requires-Dist: requests>=2.31.0
|
|
28
|
+
Requires-Dist: Flask>=3.1.0
|
|
29
|
+
Requires-Dist: Flask-HTTPAuth>=4.8.0
|
|
30
|
+
Requires-Dist: psutil>=5.9.8
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
<div align="center">
|
|
34
|
+
<h1>pyproxy</h1>
|
|
35
|
+
</div>
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
**pyproxy** is a lightweight, fast, and customizable Python-based web proxy server designed to handle both HTTP and HTTPS traffic efficiently. It can be used for various purposes, including web scraping, traffic monitoring, and content filtering.
|
|
39
|
+
|
|
40
|
+
<p align="center">
|
|
41
|
+
<img src="https://img.shields.io/github/license/6C656C65/pyproxy?style=for-the-badge">
|
|
42
|
+
<img src="https://img.shields.io/github/issues/6C656C65/pyproxy?style=for-the-badge">
|
|
43
|
+
<img src="https://img.shields.io/github/issues-closed/6C656C65/pyproxy?style=for-the-badge">
|
|
44
|
+
<br>
|
|
45
|
+
<img src="https://img.shields.io/github/forks/6C656C65/pyproxy?style=for-the-badge">
|
|
46
|
+
<img src="https://img.shields.io/github/stars/6C656C65/pyproxy?style=for-the-badge">
|
|
47
|
+
<img src="https://img.shields.io/github/commit-activity/w/6C656C65/pyproxy?style=for-the-badge">
|
|
48
|
+
<img src="https://img.shields.io/github/contributors/6C656C65/pyproxy?style=for-the-badge">
|
|
49
|
+
<br>
|
|
50
|
+
<img src="https://img.shields.io/github/actions/workflow/status/6C656C65/pyproxy/code-scan.yml?label=Scan&style=for-the-badge">
|
|
51
|
+
<img src="https://img.shields.io/github/actions/workflow/status/6C656C65/pyproxy/unittest.yml?label=Tests&style=for-the-badge">
|
|
52
|
+
<img src="https://img.shields.io/github/actions/workflow/status/6C656C65/pyproxy/docker-images.yml?label=Delivery&style=for-the-badge">
|
|
53
|
+
</p>
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## ⚡ **Features**
|
|
58
|
+
|
|
59
|
+
| Feature | Supported |
|
|
60
|
+
|----------------------------------------------|-----------|
|
|
61
|
+
| HTTP & HTTPS | ✅ |
|
|
62
|
+
| Web request logging | ✅ |
|
|
63
|
+
| Domain & URL blacklist | ✅ |
|
|
64
|
+
| SSL inspection | ✅ |
|
|
65
|
+
| Custom 403 Forbidden page | ✅ |
|
|
66
|
+
| Remote (HTTP) blacklist support | ✅ |
|
|
67
|
+
| Shortcut support | ✅ |
|
|
68
|
+
| Disable inspection for banking websites | ✅ |
|
|
69
|
+
| Custom headers | ✅ |
|
|
70
|
+
| Web interface monitoring | ✅ |
|
|
71
|
+
| Lightweight Docker image | ✅ |
|
|
72
|
+
| Proxy chaining (multi-proxy forwarding) | ✅ |
|
|
73
|
+
| IP whitelist with subnet support | ✅ |
|
|
74
|
+
|
|
75
|
+
## 📦 **Installation**
|
|
76
|
+
|
|
77
|
+
### Install from source
|
|
78
|
+
```bash
|
|
79
|
+
git clone https://github.com/6C656C65/pyproxy.git
|
|
80
|
+
cd pyproxy
|
|
81
|
+
pip install -r requirements.txt
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### Install with Docker
|
|
85
|
+
```bash
|
|
86
|
+
docker pull ghcr.io/6c656c65/pyproxy:latest
|
|
87
|
+
docker run -d ghcr.io/6c656c65/pyproxy:latest
|
|
88
|
+
```
|
|
89
|
+
You can use slim images by adding `-slim` to the end of the tags
|
|
90
|
+
|
|
91
|
+
## 🚀 **Usage**
|
|
92
|
+
|
|
93
|
+
### Start the proxy
|
|
94
|
+
```bash
|
|
95
|
+
python3 -m pyproxy.pyproxy
|
|
96
|
+
```
|
|
97
|
+
The proxy will be available at: `0.0.0.0:8080`.
|
|
98
|
+
The access log will be available at `./logs/access.log`.
|
|
99
|
+
|
|
100
|
+
## 📚 **Documentation**
|
|
101
|
+
If you encounter any problems, or if you want to use the program in a particular way, I advise you to read the [documentation](https://github.com/6C656C65/pyproxy/wiki).
|
|
102
|
+
|
|
103
|
+
## 🔧 **To do**
|
|
104
|
+
|
|
105
|
+
- Support content analysis
|
|
106
|
+
- Caching of latest and most searched pages
|
|
107
|
+
|
|
108
|
+
## 🏎️ **Benchmark**
|
|
109
|
+
|
|
110
|
+
If you're interested in benchmarking the performance of the proxy or comparing request times with and without a proxy, please refer to the [Benchmark README](benchmark/README.md) for detailed instructions on how to run the benchmarking tests and generate reports.
|
|
111
|
+
|
|
112
|
+
## 📄 **License**
|
|
113
|
+
|
|
114
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
115
|
+
|
|
116
|
+
## 🤝 **Contributing**
|
|
117
|
+
|
|
118
|
+
Contributions are welcome and appreciated! If you'd like to improve this project, feel free to fork the repository and submit a pull request. Whether it's fixing bugs, adding new features, improving documentation, or suggesting enhancements, every bit helps. Please make sure to follow the coding standards and test your changes before submitting. Let's build something great together!
|
|
119
|
+
|
|
120
|
+
## 📦 Deployment with Ansible
|
|
121
|
+
|
|
122
|
+
If you want to deploy **pyproxy** automatically to remote servers (via source or Docker), an official [Ansible role](https://github.com/6C656C65/pyproxy_ansible) is available:
|
|
123
|
+
|
|
124
|
+
* 🔧 Install from source or run as a Docker container
|
|
125
|
+
* 📁 Supports customization of ports, versions, and paths
|
|
126
|
+
* 🚀 Easily integrable into your infrastructure or CI/CD pipelines
|
|
127
|
+
|
|
128
|
+
👉 Check out the [ansible-role-pyproxy](https://github.com/6C656C65/pyproxy_ansible) repository for more details and usage instructions.
|
|
129
|
+
|
|
130
|
+
---
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
<h1>pyproxy</h1>
|
|
3
|
+
</div>
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
**pyproxy** is a lightweight, fast, and customizable Python-based web proxy server designed to handle both HTTP and HTTPS traffic efficiently. It can be used for various purposes, including web scraping, traffic monitoring, and content filtering.
|
|
7
|
+
|
|
8
|
+
<p align="center">
|
|
9
|
+
<img src="https://img.shields.io/github/license/6C656C65/pyproxy?style=for-the-badge">
|
|
10
|
+
<img src="https://img.shields.io/github/issues/6C656C65/pyproxy?style=for-the-badge">
|
|
11
|
+
<img src="https://img.shields.io/github/issues-closed/6C656C65/pyproxy?style=for-the-badge">
|
|
12
|
+
<br>
|
|
13
|
+
<img src="https://img.shields.io/github/forks/6C656C65/pyproxy?style=for-the-badge">
|
|
14
|
+
<img src="https://img.shields.io/github/stars/6C656C65/pyproxy?style=for-the-badge">
|
|
15
|
+
<img src="https://img.shields.io/github/commit-activity/w/6C656C65/pyproxy?style=for-the-badge">
|
|
16
|
+
<img src="https://img.shields.io/github/contributors/6C656C65/pyproxy?style=for-the-badge">
|
|
17
|
+
<br>
|
|
18
|
+
<img src="https://img.shields.io/github/actions/workflow/status/6C656C65/pyproxy/code-scan.yml?label=Scan&style=for-the-badge">
|
|
19
|
+
<img src="https://img.shields.io/github/actions/workflow/status/6C656C65/pyproxy/unittest.yml?label=Tests&style=for-the-badge">
|
|
20
|
+
<img src="https://img.shields.io/github/actions/workflow/status/6C656C65/pyproxy/docker-images.yml?label=Delivery&style=for-the-badge">
|
|
21
|
+
</p>
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## ⚡ **Features**
|
|
26
|
+
|
|
27
|
+
| Feature | Supported |
|
|
28
|
+
|----------------------------------------------|-----------|
|
|
29
|
+
| HTTP & HTTPS | ✅ |
|
|
30
|
+
| Web request logging | ✅ |
|
|
31
|
+
| Domain & URL blacklist | ✅ |
|
|
32
|
+
| SSL inspection | ✅ |
|
|
33
|
+
| Custom 403 Forbidden page | ✅ |
|
|
34
|
+
| Remote (HTTP) blacklist support | ✅ |
|
|
35
|
+
| Shortcut support | ✅ |
|
|
36
|
+
| Disable inspection for banking websites | ✅ |
|
|
37
|
+
| Custom headers | ✅ |
|
|
38
|
+
| Web interface monitoring | ✅ |
|
|
39
|
+
| Lightweight Docker image | ✅ |
|
|
40
|
+
| Proxy chaining (multi-proxy forwarding) | ✅ |
|
|
41
|
+
| IP whitelist with subnet support | ✅ |
|
|
42
|
+
|
|
43
|
+
## 📦 **Installation**
|
|
44
|
+
|
|
45
|
+
### Install from source
|
|
46
|
+
```bash
|
|
47
|
+
git clone https://github.com/6C656C65/pyproxy.git
|
|
48
|
+
cd pyproxy
|
|
49
|
+
pip install -r requirements.txt
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
### Install with Docker
|
|
53
|
+
```bash
|
|
54
|
+
docker pull ghcr.io/6c656c65/pyproxy:latest
|
|
55
|
+
docker run -d ghcr.io/6c656c65/pyproxy:latest
|
|
56
|
+
```
|
|
57
|
+
You can use slim images by adding `-slim` to the end of the tags
|
|
58
|
+
|
|
59
|
+
## 🚀 **Usage**
|
|
60
|
+
|
|
61
|
+
### Start the proxy
|
|
62
|
+
```bash
|
|
63
|
+
python3 -m pyproxy.pyproxy
|
|
64
|
+
```
|
|
65
|
+
The proxy will be available at: `0.0.0.0:8080`.
|
|
66
|
+
The access log will be available at `./logs/access.log`.
|
|
67
|
+
|
|
68
|
+
## 📚 **Documentation**
|
|
69
|
+
If you encounter any problems, or if you want to use the program in a particular way, I advise you to read the [documentation](https://github.com/6C656C65/pyproxy/wiki).
|
|
70
|
+
|
|
71
|
+
## 🔧 **To do**
|
|
72
|
+
|
|
73
|
+
- Support content analysis
|
|
74
|
+
- Caching of latest and most searched pages
|
|
75
|
+
|
|
76
|
+
## 🏎️ **Benchmark**
|
|
77
|
+
|
|
78
|
+
If you're interested in benchmarking the performance of the proxy or comparing request times with and without a proxy, please refer to the [Benchmark README](benchmark/README.md) for detailed instructions on how to run the benchmarking tests and generate reports.
|
|
79
|
+
|
|
80
|
+
## 📄 **License**
|
|
81
|
+
|
|
82
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
83
|
+
|
|
84
|
+
## 🤝 **Contributing**
|
|
85
|
+
|
|
86
|
+
Contributions are welcome and appreciated! If you'd like to improve this project, feel free to fork the repository and submit a pull request. Whether it's fixing bugs, adding new features, improving documentation, or suggesting enhancements, every bit helps. Please make sure to follow the coding standards and test your changes before submitting. Let's build something great together!
|
|
87
|
+
|
|
88
|
+
## 📦 Deployment with Ansible
|
|
89
|
+
|
|
90
|
+
If you want to deploy **pyproxy** automatically to remote servers (via source or Docker), an official [Ansible role](https://github.com/6C656C65/pyproxy_ansible) is available:
|
|
91
|
+
|
|
92
|
+
* 🔧 Install from source or run as a Docker container
|
|
93
|
+
* 📁 Supports customization of ports, versions, and paths
|
|
94
|
+
* 🚀 Easily integrable into your infrastructure or CI/CD pipelines
|
|
95
|
+
|
|
96
|
+
👉 Check out the [ansible-role-pyproxy](https://github.com/6C656C65/pyproxy_ansible) repository for more details and usage instructions.
|
|
97
|
+
|
|
98
|
+
---
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module provides a set of functions to benchmark the performance of a proxy server
|
|
3
|
+
by comparing the response times for HTTP requests sent with and without the use of a proxy.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import time
|
|
7
|
+
import argparse
|
|
8
|
+
import sys
|
|
9
|
+
import os
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
import pandas as pd
|
|
12
|
+
from utils.req import send_request_with_proxy, send_request_without_proxy
|
|
13
|
+
from utils.html import create_combined_html_report
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def benchmark(url: str, proxy: str, num_requests: int) -> tuple:
|
|
17
|
+
"""
|
|
18
|
+
Benchmarks the performance of sending requests to the specified$
|
|
19
|
+
URL with and without using a proxy. It sends multiple requests and
|
|
20
|
+
records the time taken for each.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
url (str): The URL to benchmark.
|
|
24
|
+
proxy (str): The proxy URL to use for the benchmark.
|
|
25
|
+
num_requests (int): The number of requests to send.
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
tuple: A tuple containing:
|
|
29
|
+
- A dictionary with statistics (average, min, max) for requests without and with proxy.
|
|
30
|
+
- A pandas DataFrame containing the times for each request without and with proxy.
|
|
31
|
+
"""
|
|
32
|
+
times_without_proxy = []
|
|
33
|
+
times_with_proxy = []
|
|
34
|
+
|
|
35
|
+
print(f"Sending requests without proxy for {url}...")
|
|
36
|
+
for i in range(num_requests):
|
|
37
|
+
times_without_proxy.append(send_request_without_proxy(url))
|
|
38
|
+
sys.stdout.write(f"\rRequests sent without proxy: {i + 1}/{num_requests}")
|
|
39
|
+
sys.stdout.flush()
|
|
40
|
+
time.sleep(0.1)
|
|
41
|
+
|
|
42
|
+
print(f"\nSending requests with proxy for {url}...")
|
|
43
|
+
for i in range(num_requests):
|
|
44
|
+
times_with_proxy.append(send_request_with_proxy(url, proxy))
|
|
45
|
+
sys.stdout.write(f"\rRequests sent with proxy: {i + 1}/{num_requests}")
|
|
46
|
+
sys.stdout.flush()
|
|
47
|
+
time.sleep(0.1)
|
|
48
|
+
|
|
49
|
+
print("\n")
|
|
50
|
+
|
|
51
|
+
stats = {
|
|
52
|
+
"avg_without_proxy": sum(times_without_proxy) / len(times_without_proxy),
|
|
53
|
+
"min_without_proxy": min(times_without_proxy),
|
|
54
|
+
"max_without_proxy": max(times_without_proxy),
|
|
55
|
+
"avg_with_proxy": sum(times_with_proxy) / len(times_with_proxy),
|
|
56
|
+
"min_with_proxy": min(times_with_proxy),
|
|
57
|
+
"max_with_proxy": max(times_with_proxy),
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
results = pd.DataFrame(
|
|
61
|
+
{
|
|
62
|
+
"Request Number": range(1, num_requests + 1),
|
|
63
|
+
"Without Proxy": times_without_proxy,
|
|
64
|
+
"With Proxy": times_with_proxy,
|
|
65
|
+
}
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
return stats, results
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def main() -> None:
|
|
72
|
+
"""
|
|
73
|
+
Main function to parse command-line arguments, run benchmarks, and generate the report.
|
|
74
|
+
It either benchmarks a single URL or a list of URLs from a file.
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
None
|
|
78
|
+
"""
|
|
79
|
+
parser = argparse.ArgumentParser(description="Proxy performance benchmark.")
|
|
80
|
+
parser.add_argument(
|
|
81
|
+
"--proxy-url",
|
|
82
|
+
type=str,
|
|
83
|
+
default="http://localhost:8080",
|
|
84
|
+
help="The proxy URL to use",
|
|
85
|
+
)
|
|
86
|
+
parser.add_argument(
|
|
87
|
+
"--target-url",
|
|
88
|
+
type=str,
|
|
89
|
+
help="A single URL to test (e.g., http://example.com)",
|
|
90
|
+
)
|
|
91
|
+
parser.add_argument(
|
|
92
|
+
"--target-file",
|
|
93
|
+
type=str,
|
|
94
|
+
help="A file containing a list of URLs to test",
|
|
95
|
+
)
|
|
96
|
+
parser.add_argument(
|
|
97
|
+
"--num-requests",
|
|
98
|
+
type=int,
|
|
99
|
+
default=10,
|
|
100
|
+
help="Number of requests to send (default: 10)",
|
|
101
|
+
)
|
|
102
|
+
parser.add_argument(
|
|
103
|
+
"--output-dir", type=str, default="benchmark/outputs", help="Output directory"
|
|
104
|
+
)
|
|
105
|
+
args = parser.parse_args()
|
|
106
|
+
|
|
107
|
+
if not args.target_url and not args.target_file:
|
|
108
|
+
print("Error: you must provide either --target-url or --target-file.")
|
|
109
|
+
sys.exit(1)
|
|
110
|
+
|
|
111
|
+
if not os.path.exists(args.output_dir):
|
|
112
|
+
os.makedirs(args.output_dir)
|
|
113
|
+
|
|
114
|
+
timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
115
|
+
all_results = {}
|
|
116
|
+
|
|
117
|
+
if args.target_file:
|
|
118
|
+
if not os.path.exists(args.target_file):
|
|
119
|
+
print(f"Error: the file {args.target_file} does not exist.")
|
|
120
|
+
sys.exit(1)
|
|
121
|
+
|
|
122
|
+
with open(args.target_file, "r", encoding="utf-8") as f:
|
|
123
|
+
urls = [line.strip() for line in f if line.strip()]
|
|
124
|
+
|
|
125
|
+
for url in urls:
|
|
126
|
+
print(f"\nBenchmarking for {url}")
|
|
127
|
+
stats, results = benchmark(url, args.proxy_url, args.num_requests)
|
|
128
|
+
all_results[url] = (stats, results)
|
|
129
|
+
else:
|
|
130
|
+
stats, results = benchmark(args.target_url, args.proxy_url, args.num_requests)
|
|
131
|
+
all_results[args.target_url] = (stats, results)
|
|
132
|
+
|
|
133
|
+
avg_without_proxy_list = []
|
|
134
|
+
avg_with_proxy_list = []
|
|
135
|
+
|
|
136
|
+
for stats, _ in all_results.values():
|
|
137
|
+
avg_without_proxy_list.append(stats["avg_without_proxy"])
|
|
138
|
+
avg_with_proxy_list.append(stats["avg_with_proxy"])
|
|
139
|
+
|
|
140
|
+
global_avg_without_proxy = sum(avg_without_proxy_list) / len(avg_without_proxy_list)
|
|
141
|
+
global_avg_with_proxy = sum(avg_with_proxy_list) / len(avg_with_proxy_list)
|
|
142
|
+
|
|
143
|
+
percentage_change = (
|
|
144
|
+
(global_avg_with_proxy - global_avg_without_proxy) / global_avg_without_proxy
|
|
145
|
+
) * 100
|
|
146
|
+
|
|
147
|
+
print(f"Global average without proxy: {global_avg_without_proxy:.6f} seconds")
|
|
148
|
+
print(f"Global average with proxy: {global_avg_with_proxy:.6f} seconds")
|
|
149
|
+
print(
|
|
150
|
+
f"Impact: {'Improvement' if percentage_change < 0 else 'Slowdown'} of "
|
|
151
|
+
f"{abs(percentage_change):.2f}%"
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
create_combined_html_report(
|
|
155
|
+
all_results,
|
|
156
|
+
global_avg_without_proxy,
|
|
157
|
+
global_avg_with_proxy,
|
|
158
|
+
percentage_change,
|
|
159
|
+
args.output_dir,
|
|
160
|
+
timestamp,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
if __name__ == "__main__":
|
|
165
|
+
main()
|
|
File without changes
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module provides functions for generating HTML reports to visualize
|
|
3
|
+
benchmark results comparing performance with and without a proxy.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import plotly.graph_objects as go
|
|
8
|
+
|
|
9
|
+
TEMPLATE_PATH = "benchmark/templates/report_template.html"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def generate_combined_table(all_results: dict) -> str:
|
|
13
|
+
"""
|
|
14
|
+
Generates a single HTML table combining statistics for all
|
|
15
|
+
URLs with sub-columns for avg, min, and max.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
all_results (dict): A dictionary containing the results for each URL.
|
|
19
|
+
|
|
20
|
+
Returns:
|
|
21
|
+
str: The HTML table as a string.
|
|
22
|
+
"""
|
|
23
|
+
table_html = """
|
|
24
|
+
<div class="summary">
|
|
25
|
+
<h2>Benchmark Results Summary</h2>
|
|
26
|
+
<table>
|
|
27
|
+
<thead>
|
|
28
|
+
<tr>
|
|
29
|
+
<th>URL</th>
|
|
30
|
+
<th colspan="3">Without Proxy</th>
|
|
31
|
+
<th colspan="3">With Proxy</th>
|
|
32
|
+
</tr>
|
|
33
|
+
<tr>
|
|
34
|
+
<th></th>
|
|
35
|
+
<th>Avg (s)</th>
|
|
36
|
+
<th>Min (s)</th>
|
|
37
|
+
<th>Max (s)</th>
|
|
38
|
+
<th>Avg (s)</th>
|
|
39
|
+
<th>Min (s)</th>
|
|
40
|
+
<th>Max (s)</th>
|
|
41
|
+
</tr>
|
|
42
|
+
</thead>
|
|
43
|
+
<tbody>
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
for url, (stats, _) in all_results.items():
|
|
47
|
+
table_html += f"""
|
|
48
|
+
<tr>
|
|
49
|
+
<td>{url}</td>
|
|
50
|
+
<td>{stats['avg_without_proxy']:.5f}</td>
|
|
51
|
+
<td>{stats['min_without_proxy']:.5f}</td>
|
|
52
|
+
<td>{stats['max_without_proxy']:.5f}</td>
|
|
53
|
+
<td>{stats['avg_with_proxy']:.5f}</td>
|
|
54
|
+
<td>{stats['min_with_proxy']:.5f}</td>
|
|
55
|
+
<td>{stats['max_with_proxy']:.5f}</td>
|
|
56
|
+
</tr>
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
table_html += """
|
|
60
|
+
</tbody>
|
|
61
|
+
</table>
|
|
62
|
+
</div>
|
|
63
|
+
<hr>
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
return table_html
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def prepare_filenames(output_dir: str, timestamp: str) -> dict:
|
|
70
|
+
"""
|
|
71
|
+
Prepares the filenames for the report and plotly files.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
output_dir (str): The directory to save the report in.
|
|
75
|
+
timestamp (str): The timestamp to use in filenames.
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
dict: A dictionary containing the plotly and html file paths.
|
|
79
|
+
"""
|
|
80
|
+
output_dir = os.path.normpath(output_dir)
|
|
81
|
+
|
|
82
|
+
plotly_filename = f"benchmark_combined_interactive_{timestamp}.html"
|
|
83
|
+
html_filename = f"benchmark_combined_report_{timestamp}.html"
|
|
84
|
+
|
|
85
|
+
plotly_filepath = os.path.join(output_dir, plotly_filename)
|
|
86
|
+
html_filepath = os.path.join(output_dir, html_filename)
|
|
87
|
+
|
|
88
|
+
return {"plotly": plotly_filepath, "html": html_filepath}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def render_template(template_path: str, context: dict) -> str:
|
|
92
|
+
"""
|
|
93
|
+
Renders an HTML template by replacing placeholders with provided context.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
template_path (str): Path to the HTML template.
|
|
97
|
+
context (dict): A dictionary with keys matching placeholders.
|
|
98
|
+
|
|
99
|
+
Returns:
|
|
100
|
+
str: The rendered HTML content.
|
|
101
|
+
"""
|
|
102
|
+
with open(template_path, "r", encoding="utf-8") as f:
|
|
103
|
+
template = f.read()
|
|
104
|
+
return template.format(**context)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def create_combined_html_report(
|
|
108
|
+
all_results: dict,
|
|
109
|
+
avg_without_proxy: float,
|
|
110
|
+
avg_with_proxy: float,
|
|
111
|
+
percentage_change: float,
|
|
112
|
+
output_dir: str,
|
|
113
|
+
timestamp: str,
|
|
114
|
+
) -> None:
|
|
115
|
+
"""
|
|
116
|
+
Generates an HTML report with the benchmark results, including graphs and statistics.
|
|
117
|
+
Saves the report to the specified output directory.
|
|
118
|
+
|
|
119
|
+
Args:
|
|
120
|
+
all_results (dict): A dictionary containing the results for each URL.
|
|
121
|
+
avg_without_proxy (float): The average time for requests without a proxy.
|
|
122
|
+
avg_with_proxy (float): The average time for requests with a proxy.
|
|
123
|
+
percentage_change (float): The percentage change in performance
|
|
124
|
+
between requests with and without a proxy.
|
|
125
|
+
output_dir (str): The directory to save the report in.
|
|
126
|
+
timestamp (str): The timestamp to use in filenames.
|
|
127
|
+
|
|
128
|
+
Returns:
|
|
129
|
+
None
|
|
130
|
+
"""
|
|
131
|
+
fig = go.Figure()
|
|
132
|
+
|
|
133
|
+
filenames = prepare_filenames(output_dir, timestamp)
|
|
134
|
+
|
|
135
|
+
for url, (_, results) in all_results.items():
|
|
136
|
+
fig.add_trace(
|
|
137
|
+
go.Scatter(
|
|
138
|
+
x=results["Request Number"],
|
|
139
|
+
y=results["Without Proxy"],
|
|
140
|
+
mode="lines+markers",
|
|
141
|
+
name=f"Without Proxy - {url}",
|
|
142
|
+
)
|
|
143
|
+
)
|
|
144
|
+
fig.add_trace(
|
|
145
|
+
go.Scatter(
|
|
146
|
+
x=results["Request Number"],
|
|
147
|
+
y=results["With Proxy"],
|
|
148
|
+
mode="lines+markers",
|
|
149
|
+
name=f"With Proxy - {url}",
|
|
150
|
+
)
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
fig.update_layout(
|
|
154
|
+
title="Response Time per Request (All URLs)",
|
|
155
|
+
xaxis_title="Request Number",
|
|
156
|
+
yaxis_title="Response Time (seconds)",
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
fig.write_html(filenames["plotly"])
|
|
160
|
+
|
|
161
|
+
html_sections = generate_combined_table(all_results)
|
|
162
|
+
|
|
163
|
+
context = {
|
|
164
|
+
"avg_without_proxy": f"{avg_without_proxy:.6f} seconds",
|
|
165
|
+
"avg_with_proxy": f"{avg_with_proxy:.6f} seconds",
|
|
166
|
+
"impact": (
|
|
167
|
+
f"{'Improvement' if percentage_change < 0 else 'Slowdown'} "
|
|
168
|
+
f"of {abs(percentage_change):.2f}%"
|
|
169
|
+
),
|
|
170
|
+
"html_sections": html_sections,
|
|
171
|
+
"plotly_filename": os.path.basename(filenames["plotly"]),
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
html_content = render_template(TEMPLATE_PATH, context)
|
|
175
|
+
|
|
176
|
+
with open(filenames["html"], "w", encoding="utf-8") as f:
|
|
177
|
+
f.write(html_content)
|
|
178
|
+
|
|
179
|
+
print(f"\nThe combined report has been generated at '{filenames['html']}'.")
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Module for sending HTTP GET requests with and without a proxy,
|
|
3
|
+
and measuring the request completion time.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import time
|
|
7
|
+
import requests
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def send_request_without_proxy(url: str) -> float:
|
|
11
|
+
"""
|
|
12
|
+
Sends an HTTP GET request to the provided URL without using a proxy,
|
|
13
|
+
and measures the time it takes to complete the request.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
url (str): The URL to send the request to.
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
float: The time taken to complete the request in seconds.
|
|
20
|
+
"""
|
|
21
|
+
start_time = time.time()
|
|
22
|
+
requests.get(url, timeout=10)
|
|
23
|
+
end_time = time.time()
|
|
24
|
+
return end_time - start_time
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def send_request_with_proxy(url: str, proxy: str) -> float:
|
|
28
|
+
"""
|
|
29
|
+
Sends an HTTP GET request to the provided URL using a proxy,
|
|
30
|
+
and measures the time it takes to complete the request.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
url (str): The URL to send the request to.
|
|
34
|
+
proxy (str): The proxy URL to use for the request.
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
float: The time taken to complete the request in seconds.
|
|
38
|
+
"""
|
|
39
|
+
proxies = {"http": proxy, "https": proxy}
|
|
40
|
+
start_time = time.time()
|
|
41
|
+
requests.get(url, proxies=proxies, timeout=10)
|
|
42
|
+
end_time = time.time()
|
|
43
|
+
return end_time - start_time
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "pyproxytools"
|
|
3
|
+
description = "Lightweight and fast python web proxy"
|
|
4
|
+
readme = "README.md"
|
|
5
|
+
license = "MIT"
|
|
6
|
+
license-files = [
|
|
7
|
+
"LICENSE",
|
|
8
|
+
]
|
|
9
|
+
authors = [{name = "6C656C65"}]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Development Status :: 5 - Production/Stable",
|
|
12
|
+
"Intended Audience :: Developers",
|
|
13
|
+
"Natural Language :: English",
|
|
14
|
+
"Operating System :: OS Independent",
|
|
15
|
+
"Programming Language :: Python :: 3.8",
|
|
16
|
+
"Programming Language :: Python :: 3.9",
|
|
17
|
+
"Programming Language :: Python :: 3.10",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Topic :: Internet",
|
|
21
|
+
"Topic :: Software Development :: Libraries",
|
|
22
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
23
|
+
"Topic :: Utilities",
|
|
24
|
+
"Typing :: Typed",
|
|
25
|
+
]
|
|
26
|
+
dynamic = ["version", "dependencies"]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Documentation = "https://github.com/6C656C65/pyproxy/wiki"
|
|
30
|
+
"Issue tracker" = "https://github.com/6C656C65/pyproxy/issues"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages]
|
|
33
|
+
find = {}
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.dynamic]
|
|
36
|
+
dependencies = { file = "requirements.txt" }
|
|
37
|
+
|
|
38
|
+
[project.scripts]
|
|
39
|
+
pyproxy = "pyproxy.pyproxy:main"
|
|
40
|
+
|
|
41
|
+
[build-system]
|
|
42
|
+
requires = ["setuptools"]
|
|
43
|
+
build-backend = "setuptools.build_meta"
|