proxy-reader 0.1.0__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of proxy-reader might be problematic. Click here for more details.
- proxy_reader/__init__.py +1 -1
- proxy_reader/_types.py +10 -1
- proxy_reader/logs_config.py +20 -0
- proxy_reader/protocols/reader.py +29 -61
- proxy_reader/proxy.py +29 -24
- proxy_reader/reader.py +89 -50
- proxy_reader/utils.py +35 -0
- {proxy_reader-0.1.0.dist-info → proxy_reader-2.0.0.dist-info}/LICENSE +0 -0
- {proxy_reader-0.1.0.dist-info → proxy_reader-2.0.0.dist-info}/METADATA +26 -5
- proxy_reader-2.0.0.dist-info/RECORD +15 -0
- {proxy_reader-0.1.0.dist-info → proxy_reader-2.0.0.dist-info}/WHEEL +1 -1
- proxy_reader/logger.py +0 -21
- proxy_reader-0.1.0.dist-info/RECORD +0 -14
- {proxy_reader-0.1.0.dist-info → proxy_reader-2.0.0.dist-info}/top_level.txt +0 -0
proxy_reader/__init__.py
CHANGED
proxy_reader/_types.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import TypedDict, NotRequired, List, Iterator, TypeAlias
|
|
1
|
+
from typing import TypedDict, NotRequired, List, Iterator, TypeAlias, Any, Optional
|
|
2
2
|
from .proxy import Proxy
|
|
3
3
|
|
|
4
4
|
|
|
@@ -10,5 +10,14 @@ class TelegramHTTP(TypedDict):
|
|
|
10
10
|
password: NotRequired[str]
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
GeneralDict: TypeAlias = dict[str, Any]
|
|
14
|
+
|
|
13
15
|
ProxiesList: TypeAlias = List[Proxy]
|
|
14
16
|
ProxyiesGen: TypeAlias = Iterator[Proxy]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ProxyDictT(TypedDict):
|
|
20
|
+
host: str
|
|
21
|
+
port: str
|
|
22
|
+
username: Optional[str]
|
|
23
|
+
password: Optional[str]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
# Create the package-wide logger
|
|
4
|
+
package_name = "proxy_reader"
|
|
5
|
+
logger = logging.getLogger(package_name)
|
|
6
|
+
logger.addHandler(logging.NullHandler())
|
|
7
|
+
logger.setLevel(logging.WARNING) # Default: Show only warnings/errors
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def enable_debug_logs() -> None:
|
|
11
|
+
logger = logging.getLogger(package_name)
|
|
12
|
+
logger.setLevel(logging.DEBUG) # Set log level
|
|
13
|
+
|
|
14
|
+
# 🔥 Make sure a handler is added (VERY IMPORTANT)
|
|
15
|
+
ch = logging.StreamHandler() # Console handler
|
|
16
|
+
ch.setLevel(logging.DEBUG) # Match the logger's level
|
|
17
|
+
formatter = logging.Formatter("%(name)s - %(levelname)s - %(message)s")
|
|
18
|
+
ch.setFormatter(formatter)
|
|
19
|
+
logger.addHandler(ch) # Attach the handler
|
|
20
|
+
logger.debug(f"Debugging enabled on {package_name!r}")
|
proxy_reader/protocols/reader.py
CHANGED
|
@@ -5,102 +5,70 @@ from ..proxy import Proxy
|
|
|
5
5
|
|
|
6
6
|
class ProxiesReaderProtocol(Protocol):
|
|
7
7
|
@property
|
|
8
|
-
def total(self) -> int:
|
|
9
|
-
pass
|
|
8
|
+
def total(self) -> int: ...
|
|
10
9
|
|
|
11
10
|
@property
|
|
12
|
-
def total_working(self) -> int:
|
|
13
|
-
pass
|
|
11
|
+
def total_working(self) -> int: ...
|
|
14
12
|
|
|
15
13
|
@property
|
|
16
|
-
def total_bad(self) -> int:
|
|
17
|
-
pass
|
|
14
|
+
def total_bad(self) -> int: ...
|
|
18
15
|
|
|
19
16
|
@property
|
|
20
|
-
def proxies(self) -> ProxiesList:
|
|
21
|
-
pass
|
|
17
|
+
def proxies(self) -> ProxiesList: ...
|
|
22
18
|
|
|
23
19
|
@property
|
|
24
|
-
def bad_proxies(self) -> ProxiesList:
|
|
25
|
-
pass
|
|
20
|
+
def bad_proxies(self) -> ProxiesList: ...
|
|
26
21
|
|
|
27
22
|
@property
|
|
28
|
-
def working_proxies(self) -> ProxiesList:
|
|
29
|
-
pass
|
|
23
|
+
def working_proxies(self) -> ProxiesList: ...
|
|
30
24
|
|
|
31
25
|
@working_proxies.setter
|
|
32
|
-
def working_proxies(self, working_proxies: List[Proxy]) -> None:
|
|
33
|
-
pass
|
|
26
|
+
def working_proxies(self, working_proxies: List[Proxy]) -> None: ...
|
|
34
27
|
|
|
35
|
-
def
|
|
36
|
-
pass
|
|
28
|
+
def _random_proxy_check_url(self) -> str: ...
|
|
37
29
|
|
|
38
|
-
def
|
|
39
|
-
pass
|
|
30
|
+
def read_with_auth(self) -> None: ...
|
|
40
31
|
|
|
41
|
-
def
|
|
42
|
-
pass
|
|
43
|
-
|
|
44
|
-
def read_authless(self) -> None:
|
|
45
|
-
pass
|
|
32
|
+
def read_authless(self) -> None: ...
|
|
46
33
|
|
|
47
34
|
async def _check_proxy(
|
|
48
35
|
self, proxy: Proxy, response_time: Optional[int] = None
|
|
49
|
-
) -> bool:
|
|
50
|
-
pass
|
|
36
|
+
) -> bool: ...
|
|
51
37
|
|
|
52
|
-
async def check_all_proxies(self, max_resp_time: int = 30) -> None:
|
|
53
|
-
pass
|
|
38
|
+
async def check_all_proxies(self, max_resp_time: int = 30) -> None: ...
|
|
54
39
|
|
|
55
40
|
async def _check_proxy_socks(
|
|
56
41
|
self, proxy: Proxy, response_time: Optional[int] = None
|
|
57
|
-
) -> bool:
|
|
58
|
-
pass
|
|
42
|
+
) -> bool: ...
|
|
59
43
|
|
|
60
|
-
async def check_all_proxies_socks5(self, max_resp_time: int = 5) -> None:
|
|
61
|
-
pass
|
|
44
|
+
async def check_all_proxies_socks5(self, max_resp_time: int = 5) -> None: ...
|
|
62
45
|
|
|
63
|
-
def get_working_proxies_list_http(self) -> List[str]:
|
|
64
|
-
pass
|
|
46
|
+
def get_working_proxies_list_http(self) -> List[str]: ...
|
|
65
47
|
|
|
66
|
-
def write_working_proxies(self, filename: str) -> None:
|
|
67
|
-
pass
|
|
48
|
+
def write_working_proxies(self, filename: str) -> None: ...
|
|
68
49
|
|
|
69
|
-
def get_random_http(self) -> Optional[str]:
|
|
70
|
-
pass
|
|
50
|
+
def get_random_http(self) -> Optional[str]: ...
|
|
71
51
|
|
|
72
|
-
def get_random_socks5(self) -> Optional[str]:
|
|
73
|
-
pass
|
|
52
|
+
def get_random_socks5(self) -> Optional[str]: ...
|
|
74
53
|
|
|
75
|
-
def get_random_socks5_telegram(self) -> Optional[Dict[str, Any]]:
|
|
76
|
-
pass
|
|
54
|
+
def get_random_socks5_telegram(self) -> Optional[Dict[str, Any]]: ...
|
|
77
55
|
|
|
78
|
-
def next_http_from_list(self) -> Optional[str]:
|
|
79
|
-
pass
|
|
56
|
+
def next_http_from_list(self) -> Optional[str]: ...
|
|
80
57
|
|
|
81
|
-
def next_http_from_cycle(self) -> str:
|
|
82
|
-
pass
|
|
58
|
+
def next_http_from_cycle(self) -> str: ...
|
|
83
59
|
|
|
84
|
-
def next_socks5_from_list(self) -> str:
|
|
85
|
-
pass
|
|
60
|
+
def next_socks5_from_list(self) -> str: ...
|
|
86
61
|
|
|
87
|
-
def next_socks5_from_cycle(self) -> str:
|
|
88
|
-
pass
|
|
62
|
+
def next_socks5_from_cycle(self) -> str: ...
|
|
89
63
|
|
|
90
|
-
def next_http_telegram_from_list(self) -> Dict[str, Any]:
|
|
91
|
-
pass
|
|
64
|
+
def next_http_telegram_from_list(self) -> Dict[str, Any]: ...
|
|
92
65
|
|
|
93
|
-
def next_http_telegram_from_cycle(self) -> Dict[str, Any]:
|
|
94
|
-
pass
|
|
66
|
+
def next_http_telegram_from_cycle(self) -> Dict[str, Any]: ...
|
|
95
67
|
|
|
96
|
-
def next_socks5_telegram_from_cycle(self) -> Dict[str, Any]:
|
|
97
|
-
pass
|
|
68
|
+
def next_socks5_telegram_from_cycle(self) -> Dict[str, Any]: ...
|
|
98
69
|
|
|
99
|
-
def next_socks5_telegram_from_list(self) -> Dict[str, Any]:
|
|
100
|
-
pass
|
|
70
|
+
def next_socks5_telegram_from_list(self) -> Dict[str, Any]: ...
|
|
101
71
|
|
|
102
|
-
def next_https_from_list(self) -> str:
|
|
103
|
-
pass
|
|
72
|
+
def next_https_from_list(self) -> str: ...
|
|
104
73
|
|
|
105
|
-
def next_https_from_cycle(self) -> str:
|
|
106
|
-
pass
|
|
74
|
+
def next_https_from_cycle(self) -> str: ...
|
proxy_reader/proxy.py
CHANGED
|
@@ -1,78 +1,83 @@
|
|
|
1
|
-
from
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from typing import TYPE_CHECKING, Optional, Dict, Any
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
if TYPE_CHECKING:
|
|
6
|
+
from ._types import GeneralDict, ProxyDictT
|
|
2
7
|
|
|
3
8
|
|
|
4
9
|
class Proxy:
|
|
5
|
-
def __init__(
|
|
6
|
-
self
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
self.
|
|
10
|
+
def __init__(
|
|
11
|
+
self,
|
|
12
|
+
proxy: ProxyDictT,
|
|
13
|
+
) -> None:
|
|
14
|
+
self._proxy = proxy
|
|
10
15
|
|
|
11
16
|
@property
|
|
12
17
|
def ip(self) -> str:
|
|
13
|
-
return self.
|
|
18
|
+
return self._proxy["host"]
|
|
14
19
|
|
|
15
20
|
@property
|
|
16
21
|
def port(self) -> str:
|
|
17
|
-
return self.
|
|
22
|
+
return self._proxy["port"]
|
|
18
23
|
|
|
19
24
|
@property
|
|
20
25
|
def username(self) -> Optional[str]:
|
|
21
|
-
return self.
|
|
26
|
+
return self._proxy["username"]
|
|
22
27
|
|
|
23
28
|
@property
|
|
24
29
|
def password(self) -> Optional[str]:
|
|
25
|
-
return self.
|
|
30
|
+
return self._proxy["password"]
|
|
26
31
|
|
|
27
32
|
@property
|
|
28
33
|
def http(self) -> str:
|
|
29
34
|
if self.username and self.password:
|
|
30
|
-
return f"http://{self.
|
|
31
|
-
return f"http://{self.
|
|
35
|
+
return f"http://{self.username}:{self.password}@{self.ip}:{self.port}"
|
|
36
|
+
return f"http://{self.ip}:{self.port}"
|
|
32
37
|
|
|
33
38
|
@property
|
|
34
39
|
def https(self) -> str:
|
|
35
40
|
if self.username and self.password:
|
|
36
|
-
return f"https://{self.
|
|
37
|
-
return f"https://{self.
|
|
41
|
+
return f"https://{self.username}:{self.password}@{self.ip}:{self.port}"
|
|
42
|
+
return f"https://{self.ip}:{self.port}"
|
|
38
43
|
|
|
39
44
|
@property
|
|
40
45
|
def telegram_http(self) -> Dict[str, Any]:
|
|
41
|
-
p = {"proxy_type": 3, "addr": self.
|
|
46
|
+
p: GeneralDict = {"proxy_type": 3, "addr": self.ip, "port": int(self.port)}
|
|
42
47
|
if self.username and self.password:
|
|
43
48
|
p.update({"username": self.username, "password": self.password})
|
|
44
49
|
return p
|
|
45
50
|
|
|
46
51
|
@property
|
|
47
52
|
def telegram_socks5(self) -> Dict[str, Any]:
|
|
48
|
-
p = {"proxy_type": 2, "addr": self.
|
|
53
|
+
p: GeneralDict = {"proxy_type": 2, "addr": self.ip, "port": int(self.port)}
|
|
49
54
|
if self.username and self.password:
|
|
50
55
|
p.update({"username": self.username, "password": self.password})
|
|
51
56
|
return p
|
|
52
57
|
|
|
53
58
|
@property
|
|
54
59
|
def telegram_socks4(self) -> Dict[str, Any]:
|
|
55
|
-
p = {"proxy_type": 1, "addr": self.
|
|
60
|
+
p: GeneralDict = {"proxy_type": 1, "addr": self.ip, "port": int(self.port)}
|
|
56
61
|
if self.username and self.password:
|
|
57
62
|
p.update({"username": self.username, "password": self.password})
|
|
58
63
|
return p
|
|
59
64
|
|
|
60
65
|
@property
|
|
61
66
|
def socks5(self) -> str:
|
|
62
|
-
if self.
|
|
63
|
-
return f"socks5://{self.
|
|
67
|
+
if self.username is not None and self.password is not None:
|
|
68
|
+
return f"socks5://{self.username}:{self.password}@{self.ip}:{self.port}"
|
|
64
69
|
return f"socks5://{self.ip}:{self.port}"
|
|
65
70
|
|
|
66
71
|
@property
|
|
67
72
|
def socks4(self) -> str:
|
|
68
|
-
if self.
|
|
69
|
-
return f"socks4://{self.
|
|
73
|
+
if self.username is not None and self.password is not None:
|
|
74
|
+
return f"socks4://{self.username}:{self.password}@{self.ip}:{self.port}"
|
|
70
75
|
return f"socks4://{self.ip}:{self.port}"
|
|
71
76
|
|
|
72
77
|
def __str__(self) -> str:
|
|
73
|
-
if self.
|
|
74
|
-
return f"{self.
|
|
75
|
-
return f"{self.
|
|
78
|
+
if self.username is not None and self.password is not None:
|
|
79
|
+
return f"{self.ip}:{self.port}:{self.username}:{self.password}"
|
|
80
|
+
return f"{self.ip}:{self.port}"
|
|
76
81
|
|
|
77
82
|
def __repr__(self) -> str:
|
|
78
83
|
return self.__str__()
|
proxy_reader/reader.py
CHANGED
|
@@ -1,40 +1,52 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import random
|
|
2
3
|
import itertools
|
|
4
|
+
import tempfile
|
|
5
|
+
import warnings
|
|
6
|
+
from .utils import parse_proxy_line
|
|
3
7
|
from .proxy import Proxy
|
|
4
8
|
import aiohttp
|
|
5
9
|
import asyncio
|
|
6
|
-
from .
|
|
7
|
-
import os
|
|
10
|
+
from .logs_config import logger
|
|
8
11
|
from aiohttp_socks import ProxyConnector
|
|
9
12
|
import sys
|
|
10
13
|
from typing import Optional, List, Dict, Any
|
|
11
|
-
from ._types import ProxiesList, ProxyiesGen
|
|
14
|
+
from ._types import ProxiesList, ProxyDictT, ProxyiesGen
|
|
12
15
|
from .protocols.reader import ProxiesReaderProtocol
|
|
13
16
|
|
|
14
17
|
|
|
15
18
|
class ProxiesReader(ProxiesReaderProtocol):
|
|
16
19
|
def __init__(
|
|
17
20
|
self,
|
|
18
|
-
|
|
21
|
+
proxies_file: str,
|
|
22
|
+
check_proxies: bool = False,
|
|
23
|
+
proxy_checking_threads: int = 50,
|
|
24
|
+
max_response_time: int = 60,
|
|
19
25
|
shuffle: bool = False,
|
|
20
|
-
debug: bool = False,
|
|
21
|
-
extra_debug: bool = False,
|
|
22
26
|
) -> None:
|
|
23
|
-
self.
|
|
24
|
-
self._debug = debug
|
|
25
|
-
self._extra_debug = extra_debug
|
|
27
|
+
self._raw_proxies = open(proxies_file, encoding="utf-8", mode="r").readlines()
|
|
26
28
|
|
|
29
|
+
self._check_proxies = check_proxies
|
|
27
30
|
self._shuffle = shuffle
|
|
28
|
-
self.
|
|
29
|
-
|
|
31
|
+
self._proxies_dict_list: list[ProxyDictT] = [
|
|
32
|
+
parse_proxy_line(p) for p in self._raw_proxies
|
|
33
|
+
]
|
|
34
|
+
self._all_proxies: ProxiesList = [Proxy(p) for p in self._proxies_dict_list]
|
|
35
|
+
|
|
30
36
|
self._bad_proxies: ProxiesList = []
|
|
31
|
-
|
|
32
|
-
self.
|
|
37
|
+
|
|
38
|
+
self._working_proxies: ProxiesList = (
|
|
39
|
+
[] if self._check_proxies else self._all_proxies
|
|
40
|
+
)
|
|
41
|
+
|
|
33
42
|
self._proxy_iterator: Optional[ProxyiesGen] = None
|
|
34
43
|
self._proxy_iterator_cycle: Optional[ProxyiesGen] = None
|
|
35
|
-
self._thread_control: asyncio.Semaphore = asyncio.Semaphore(
|
|
36
|
-
|
|
44
|
+
self._thread_control: asyncio.Semaphore = asyncio.Semaphore(
|
|
45
|
+
proxy_checking_threads
|
|
46
|
+
)
|
|
47
|
+
self._max_response_time = max_response_time
|
|
37
48
|
self._timeout_count = 0
|
|
49
|
+
self._proxies_checked = False
|
|
38
50
|
|
|
39
51
|
self._check_urls = [
|
|
40
52
|
# New
|
|
@@ -53,17 +65,39 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
53
65
|
# "http://dog.ceo/api/breeds/image/random",
|
|
54
66
|
]
|
|
55
67
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
68
|
+
@classmethod
|
|
69
|
+
def load_list(
|
|
70
|
+
cls,
|
|
71
|
+
proxies: list[str],
|
|
72
|
+
check_proxies: bool = False,
|
|
73
|
+
proxy_checking_threads: int = 50,
|
|
74
|
+
max_response_time: int = 60,
|
|
75
|
+
shuffle: bool = False,
|
|
76
|
+
) -> "ProxiesReader":
|
|
77
|
+
"""Load proxies from a list"""
|
|
78
|
+
proxies_file = tempfile.NamedTemporaryFile(
|
|
79
|
+
delete=False, suffix=".txt", mode="w+"
|
|
80
|
+
)
|
|
81
|
+
try:
|
|
82
|
+
proxies_file.write("\n".join(proxies))
|
|
83
|
+
proxies_file.flush()
|
|
84
|
+
new = cls(
|
|
85
|
+
proxies_file.name,
|
|
86
|
+
check_proxies,
|
|
87
|
+
proxy_checking_threads,
|
|
88
|
+
max_response_time,
|
|
89
|
+
shuffle,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
finally:
|
|
93
|
+
proxies_file.close()
|
|
94
|
+
os.unlink(proxies_file.name)
|
|
95
|
+
|
|
96
|
+
return new
|
|
63
97
|
|
|
64
98
|
@property
|
|
65
99
|
def total(self) -> int:
|
|
66
|
-
return len(self.
|
|
100
|
+
return len(self._all_proxies)
|
|
67
101
|
|
|
68
102
|
@property
|
|
69
103
|
def total_working(self) -> int:
|
|
@@ -75,7 +109,7 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
75
109
|
|
|
76
110
|
@property
|
|
77
111
|
def proxies(self) -> ProxiesList:
|
|
78
|
-
return self.
|
|
112
|
+
return self._all_proxies
|
|
79
113
|
|
|
80
114
|
@property
|
|
81
115
|
def bad_proxies(self) -> ProxiesList:
|
|
@@ -90,46 +124,48 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
90
124
|
self._working_proxies = working_proxies
|
|
91
125
|
|
|
92
126
|
def __str__(self) -> str:
|
|
93
|
-
return str(self.
|
|
127
|
+
return str(self._all_proxies)
|
|
94
128
|
|
|
95
129
|
def __repr__(self) -> str:
|
|
96
130
|
return self.__str__()
|
|
97
131
|
|
|
98
|
-
def
|
|
99
|
-
lines = open(self._file_path).readlines()
|
|
100
|
-
return [line.strip().replace("\n", "") for line in lines]
|
|
101
|
-
|
|
102
|
-
def random_url(self) -> str:
|
|
132
|
+
def _random_proxy_check_url(self) -> str:
|
|
103
133
|
return random.choice(self._check_urls)
|
|
104
134
|
|
|
105
135
|
def read_with_auth(self) -> None:
|
|
106
136
|
"""Format: IP:PORT:USERNAME:PASSWORD"""
|
|
107
|
-
|
|
137
|
+
return warnings.warn(
|
|
138
|
+
"read_with_auth is deprecated. Please don't use it. It's not working ...",
|
|
139
|
+
)
|
|
140
|
+
raw_proxies = self._read_raw()
|
|
108
141
|
for proxy in raw_proxies:
|
|
109
142
|
sp_proxy = proxy.split(":")
|
|
110
143
|
ip = sp_proxy[0]
|
|
111
144
|
port = sp_proxy[1]
|
|
112
145
|
username = sp_proxy[2]
|
|
113
146
|
password = sp_proxy[3]
|
|
114
|
-
self.
|
|
147
|
+
self._all_proxies.append(Proxy(ip, port, username, password))
|
|
115
148
|
|
|
116
149
|
self._has_auth = True
|
|
117
150
|
if self._shuffle:
|
|
118
|
-
random.shuffle(self.
|
|
151
|
+
random.shuffle(self._all_proxies)
|
|
119
152
|
|
|
120
153
|
def read_authless(self) -> None:
|
|
121
154
|
"""Format: IP:PORT"""
|
|
122
|
-
|
|
155
|
+
return warnings.warn(
|
|
156
|
+
"read_authless is deprecated. Please don't use it. It's not working ..."
|
|
157
|
+
)
|
|
158
|
+
raw_proxies = self._read_raw()
|
|
123
159
|
for proxy in raw_proxies:
|
|
124
160
|
sp_proxy = proxy.split(":")
|
|
125
161
|
ip = sp_proxy[0]
|
|
126
162
|
port = sp_proxy[1]
|
|
127
|
-
self.
|
|
163
|
+
self._all_proxies.append(Proxy(ip, port))
|
|
128
164
|
logger.debug(
|
|
129
|
-
f"Loaded total {len(self.
|
|
165
|
+
f"Loaded total {len(self._all_proxies)} proxies from {self._file_path}"
|
|
130
166
|
)
|
|
131
167
|
if self._shuffle:
|
|
132
|
-
random.shuffle(self.
|
|
168
|
+
random.shuffle(self._all_proxies)
|
|
133
169
|
|
|
134
170
|
async def _check_proxy(
|
|
135
171
|
self, proxy: Proxy, response_time: Optional[int] = None
|
|
@@ -137,7 +173,7 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
137
173
|
connectins_limit = 60 if "win" in sys.platform else 100
|
|
138
174
|
connector = aiohttp.TCPConnector(limit=connectins_limit)
|
|
139
175
|
session = aiohttp.ClientSession(connector=connector)
|
|
140
|
-
url = self.
|
|
176
|
+
url = self._random_proxy_check_url()
|
|
141
177
|
p = proxy.http
|
|
142
178
|
|
|
143
179
|
async with self._thread_control:
|
|
@@ -156,14 +192,17 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
156
192
|
self._timeout_count += 1
|
|
157
193
|
logger.debug(f"{p} : TIMEOUT {e}. {url}")
|
|
158
194
|
self._bad_proxies.append(proxy)
|
|
195
|
+
await connector.close()
|
|
159
196
|
await session.close()
|
|
160
197
|
return False
|
|
161
198
|
|
|
162
199
|
except Exception as e:
|
|
163
|
-
logger.debug(f"Bad proxy raised. {e}", exc_info=
|
|
164
|
-
await session.close()
|
|
200
|
+
logger.debug(f"Bad proxy raised. {e}", exc_info=True)
|
|
165
201
|
return False
|
|
166
202
|
|
|
203
|
+
finally:
|
|
204
|
+
await session.close()
|
|
205
|
+
|
|
167
206
|
if resp.status == 200:
|
|
168
207
|
logger.debug(f"{p}: Working")
|
|
169
208
|
self._working_proxies.append(proxy)
|
|
@@ -176,19 +215,19 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
176
215
|
|
|
177
216
|
async def check_all_proxies(self, max_resp_time: int = 30) -> None:
|
|
178
217
|
"""Run this to check all proxies at once."""
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
218
|
+
if self._check_proxies:
|
|
219
|
+
async with asyncio.TaskGroup() as gp:
|
|
220
|
+
for proxy in self._all_proxies:
|
|
221
|
+
gp.create_task(self._check_proxy(proxy, max_resp_time))
|
|
222
|
+
self._proxies_checked = True
|
|
223
|
+
logger.debug("All proxies checked.")
|
|
185
224
|
|
|
186
225
|
async def _check_proxy_socks(
|
|
187
226
|
self, proxy: Proxy, response_time: Optional[int] = None
|
|
188
227
|
) -> bool:
|
|
189
|
-
url = self.
|
|
228
|
+
url = self._random_proxy_check_url()
|
|
190
229
|
socks_connector = ProxyConnector.from_url(proxy.socks5) # type: ignore
|
|
191
|
-
session = aiohttp.ClientSession(connector=socks_connector)
|
|
230
|
+
session = aiohttp.ClientSession(connector=socks_connector) # type: ignore
|
|
192
231
|
logger.debug(f"Checking proxy {proxy} ..")
|
|
193
232
|
try:
|
|
194
233
|
resp = await asyncio.wait_for(session.get(url), timeout=response_time)
|
|
@@ -200,7 +239,7 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
200
239
|
return False
|
|
201
240
|
|
|
202
241
|
except Exception as e:
|
|
203
|
-
logger.debug(f"Bad proxy raised. {e}", exc_info=
|
|
242
|
+
logger.debug(f"Bad proxy raised. {e}", exc_info=True)
|
|
204
243
|
await session.close()
|
|
205
244
|
return False
|
|
206
245
|
|
|
@@ -218,7 +257,7 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
218
257
|
async def check_all_proxies_socks5(self, max_resp_time: int = 5) -> None:
|
|
219
258
|
"""Run the check on all proxies at once."""
|
|
220
259
|
tasks: List[asyncio.Task[bool]] = []
|
|
221
|
-
for proxy in self.
|
|
260
|
+
for proxy in self._all_proxies:
|
|
222
261
|
tasks.append(
|
|
223
262
|
asyncio.create_task(self._check_proxy_socks(proxy, max_resp_time))
|
|
224
263
|
)
|
proxy_reader/utils.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from ._types import ProxyDictT
|
|
2
|
+
import re
|
|
3
|
+
from typing import cast
|
|
4
|
+
from .logs_config import logger
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def parse_proxy_line(proxy: str) -> ProxyDictT:
|
|
8
|
+
"""
|
|
9
|
+
Detects the format of the proxy string and extracts the components.
|
|
10
|
+
|
|
11
|
+
Returns a dictionary with 'ip_or_host', 'port', 'username', and 'password'.
|
|
12
|
+
"""
|
|
13
|
+
logger.info("Reading the proxy format ...")
|
|
14
|
+
patterns: list[str] = [
|
|
15
|
+
# Format 1: IP/Hostname:PORT:USERNAME:PASSWORD
|
|
16
|
+
r"^(?P<host>[\w\.-]+):(?P<port>\d+):(?P<username>[^:]+):(?P<password>.+)$",
|
|
17
|
+
# Format 2: USERNAME:PASSWORD:IP/Hostname:PORT
|
|
18
|
+
r"^(?P<username>[^:]+):(?P<password>[^:]+):(?P<host>[\w\.-]+):(?P<port>\d+)$",
|
|
19
|
+
# Format 3: http://USERNAME:PASSWORD@IP/Hostname:PORT
|
|
20
|
+
r"^http:\/\/(?P<username>[^:]+):(?P<password>[^@]+)@(?P<host>[\w\.-]+):(?P<port>\d+)$",
|
|
21
|
+
# Format 4: USERNAME:PASSWORD@IP/Hostname:PORT
|
|
22
|
+
r"^(?P<username>[^:]+):(?P<password>[^@]+)@(?P<host>[\w\.-]+):(?P<port>\d+)$",
|
|
23
|
+
# Format 5: IP/Hostname:PORT (No Username/Password)
|
|
24
|
+
r"^(?P<ip_or_host>[\w\.-]+):(?P<port>\d+)$",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
for pattern in patterns:
|
|
28
|
+
match = re.match(pattern, proxy.strip())
|
|
29
|
+
if match:
|
|
30
|
+
data = match.groupdict()
|
|
31
|
+
data.setdefault("username", None) # Ensure username exists
|
|
32
|
+
data.setdefault("password", None) # Ensure password exists
|
|
33
|
+
return cast(ProxyDictT, data)
|
|
34
|
+
|
|
35
|
+
raise ValueError(f"Invalid format: {proxy}")
|
|
File without changes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: proxy-reader
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: Read and check bulk proxies effectively
|
|
5
5
|
Author-email: Rune Tech <runetech2024@gmail.com>
|
|
6
6
|
License: The MIT License (MIT)
|
|
@@ -17,17 +17,38 @@ Keywords: proxies,proxy,reader,checker,bulk
|
|
|
17
17
|
Requires-Python: >=3.11
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
|
-
Requires-Dist: aiohttp
|
|
21
|
-
Requires-Dist:
|
|
20
|
+
Requires-Dist: aiohttp>=3.10.10
|
|
21
|
+
Requires-Dist: aiohttp_socks>=0.9.0
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
#
|
|
26
|
+
# proxy-reader - A simple but useful bulk proxies reader and checker.
|
|
27
27
|
|
|
28
28
|
This is useful when you are working with multiple proxies and want to bulk check
|
|
29
29
|
these proxies before using them.
|
|
30
30
|
It has iterators to iterate the proxies for easy reading.
|
|
31
31
|
|
|
32
32
|
|
|
33
|
+
## Installation
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
pip install proxy-reader
|
|
37
|
+
```
|
|
38
|
+
|
|
33
39
|
## Changelogs
|
|
40
|
+
|
|
41
|
+
### [v0.3.0]
|
|
42
|
+
* Add load_list class method to load proxies from list
|
|
43
|
+
* MAJOR: auto detect the proxies format
|
|
44
|
+
* Use new logging config to disable logs and prevent proxy_reader.log from creating
|
|
45
|
+
* Logs can be enabled from user-side
|
|
46
|
+
* Add option to disable proxies checking from __init__
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
### [v0.2.0] - 2024-10-21
|
|
50
|
+
|
|
51
|
+
* Previous unrecorded changes
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
For more info/queries Telegram: [@runetech](https://t.me/runetech)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
proxy_reader/__init__.py,sha256=vfiYXTHuR6qdMnj59SpTDv8lfyhX7BXRCJz16ejC3QI,46
|
|
2
|
+
proxy_reader/_types.py,sha256=C285ga4m1KGJ9IREBGzXp6bdOIOm87rJKu2USQ1tZG0,488
|
|
3
|
+
proxy_reader/domains.py,sha256=d4FUb2zw3Kg9bAjtaggEcxZA293WvVcRTgMKKPmYr1A,135662
|
|
4
|
+
proxy_reader/logs_config.py,sha256=SXNDi0op7Bk69qxbVsYajdSAfhhw9eTVVdf6MS2v6Ts,747
|
|
5
|
+
proxy_reader/proxy.py,sha256=R_imEwRmuHoflSsY5-SL6x840aRAGswwSYQkjUEF0MU,2676
|
|
6
|
+
proxy_reader/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
proxy_reader/reader.py,sha256=-TM1yyZJm4KjayzYuxVwuXYSJ4AUIMimlwdxP41cNSY,13563
|
|
8
|
+
proxy_reader/utils.py,sha256=WI-yVQF_Jjz9z-ougsHJgoXGC9fTqWwWHfwd1-7HUh4,1462
|
|
9
|
+
proxy_reader/protocols/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
+
proxy_reader/protocols/reader.py,sha256=UbQKOPhIqRwaGpgN29-zdUs1hYOaBKCyqjgb9gQHrD0,2004
|
|
11
|
+
proxy_reader-2.0.0.dist-info/LICENSE,sha256=8DyHi9RsZXeM8s1DpaXubZSSG44PkzV2oskJE3E09F4,1081
|
|
12
|
+
proxy_reader-2.0.0.dist-info/METADATA,sha256=SUjVd02zim8EpqL5pRtrCHUgLakuIVMh0qfsGS8r4bQ,2366
|
|
13
|
+
proxy_reader-2.0.0.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
|
|
14
|
+
proxy_reader-2.0.0.dist-info/top_level.txt,sha256=NIrKvLf5DF2_oEnj0yg84roFna2-MaJ7zE1jY2CIU0I,13
|
|
15
|
+
proxy_reader-2.0.0.dist-info/RECORD,,
|
proxy_reader/logger.py
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
import logging
|
|
4
|
-
from logging.handlers import RotatingFileHandler
|
|
5
|
-
|
|
6
|
-
filename = "proxy_reader.log"
|
|
7
|
-
logging_format = logging.Formatter(
|
|
8
|
-
"%(levelname)s:[%(filename)s:%(lineno)s]:%(asctime)s: %(message)s")
|
|
9
|
-
logger = logging.getLogger(__name__)
|
|
10
|
-
console_handler = logging.StreamHandler()
|
|
11
|
-
console_handler.setFormatter(logging_format)
|
|
12
|
-
file_handler = RotatingFileHandler(filename, mode='a', maxBytes=5 * 1024 * 1024,
|
|
13
|
-
backupCount=2, encoding="utf-8", delay=False)
|
|
14
|
-
file_handler.setFormatter(logging_format)
|
|
15
|
-
|
|
16
|
-
logger.setLevel(logging.DEBUG)
|
|
17
|
-
file_handler.setLevel(logging.DEBUG)
|
|
18
|
-
console_handler.setLevel(logging.DEBUG)
|
|
19
|
-
|
|
20
|
-
logger.addHandler(console_handler)
|
|
21
|
-
logger.addHandler(file_handler)
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
proxy_reader/__init__.py,sha256=8_HgszorTyU_0OJmvxgimP9S21y07s9R1n_U-wHMwTs,46
|
|
2
|
-
proxy_reader/_types.py,sha256=OWAYiN91E_IWgs1myPy1Q9l-8XkVVq8uDzjLOTB_Eww,317
|
|
3
|
-
proxy_reader/domains.py,sha256=d4FUb2zw3Kg9bAjtaggEcxZA293WvVcRTgMKKPmYr1A,135662
|
|
4
|
-
proxy_reader/logger.py,sha256=BxaHptwZlPPGdx5mbS8vi77u0N-W6cKhcR-5lG2x93c,710
|
|
5
|
-
proxy_reader/proxy.py,sha256=vkI2vonxlm0x0u3Tojq6xKaw5_7fvSbmICW8b_M4lzY,2645
|
|
6
|
-
proxy_reader/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
-
proxy_reader/reader.py,sha256=KeaAPhxMI6moYaLmzYj7ZGcn8REXWs-1xBMkXuK2Ff0,12373
|
|
8
|
-
proxy_reader/protocols/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
-
proxy_reader/protocols/reader.py,sha256=WscEXPpKmJuMF2R7EBqDZTheRh8BIUilukojcPRFyRg,2303
|
|
10
|
-
proxy_reader-0.1.0.dist-info/LICENSE,sha256=8DyHi9RsZXeM8s1DpaXubZSSG44PkzV2oskJE3E09F4,1081
|
|
11
|
-
proxy_reader-0.1.0.dist-info/METADATA,sha256=hsWGBfcZrYHMTZWKY_oQEkWBgYzisQ66sWJkt0AM5JI,1892
|
|
12
|
-
proxy_reader-0.1.0.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
|
|
13
|
-
proxy_reader-0.1.0.dist-info/top_level.txt,sha256=NIrKvLf5DF2_oEnj0yg84roFna2-MaJ7zE1jY2CIU0I,13
|
|
14
|
-
proxy_reader-0.1.0.dist-info/RECORD,,
|
|
File without changes
|