proxy-reader 0.1.0__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of proxy-reader might be problematic. Click here for more details.
- proxy_reader/__init__.py +1 -2
- proxy_reader/_types.py +10 -1
- proxy_reader/logs_config.py +20 -0
- proxy_reader/protocols/reader.py +29 -61
- proxy_reader/proxy.py +29 -24
- proxy_reader/reader.py +112 -71
- proxy_reader/utils.py +36 -0
- proxy_reader-2.1.0.dist-info/METADATA +48 -0
- proxy_reader-2.1.0.dist-info/RECORD +15 -0
- {proxy_reader-0.1.0.dist-info → proxy_reader-2.1.0.dist-info}/WHEEL +1 -1
- {proxy_reader-0.1.0.dist-info → proxy_reader-2.1.0.dist-info/licenses}/LICENSE +0 -0
- proxy_reader/logger.py +0 -21
- proxy_reader-0.1.0.dist-info/METADATA +0 -33
- proxy_reader-0.1.0.dist-info/RECORD +0 -14
- {proxy_reader-0.1.0.dist-info → proxy_reader-2.1.0.dist-info}/top_level.txt +0 -0
proxy_reader/__init__.py
CHANGED
proxy_reader/_types.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import TypedDict, NotRequired, List, Iterator, TypeAlias
|
|
1
|
+
from typing import TypedDict, NotRequired, List, Iterator, TypeAlias, Any, Optional
|
|
2
2
|
from .proxy import Proxy
|
|
3
3
|
|
|
4
4
|
|
|
@@ -10,5 +10,14 @@ class TelegramHTTP(TypedDict):
|
|
|
10
10
|
password: NotRequired[str]
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
GeneralDict: TypeAlias = dict[str, Any]
|
|
14
|
+
|
|
13
15
|
ProxiesList: TypeAlias = List[Proxy]
|
|
14
16
|
ProxyiesGen: TypeAlias = Iterator[Proxy]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ProxyDictT(TypedDict):
|
|
20
|
+
host: str
|
|
21
|
+
port: str
|
|
22
|
+
username: Optional[str]
|
|
23
|
+
password: Optional[str]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
# Create the package-wide logger
|
|
4
|
+
package_name = "proxy_reader"
|
|
5
|
+
logger = logging.getLogger(package_name)
|
|
6
|
+
logger.addHandler(logging.NullHandler())
|
|
7
|
+
logger.setLevel(logging.WARNING) # Default: Show only warnings/errors
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def enable_debug_logs() -> None:
|
|
11
|
+
logger = logging.getLogger(package_name)
|
|
12
|
+
logger.setLevel(logging.DEBUG) # Set log level
|
|
13
|
+
|
|
14
|
+
# 🔥 Make sure a handler is added (VERY IMPORTANT)
|
|
15
|
+
ch = logging.StreamHandler() # Console handler
|
|
16
|
+
ch.setLevel(logging.DEBUG) # Match the logger's level
|
|
17
|
+
formatter = logging.Formatter("%(name)s - %(levelname)s - %(message)s")
|
|
18
|
+
ch.setFormatter(formatter)
|
|
19
|
+
logger.addHandler(ch) # Attach the handler
|
|
20
|
+
logger.debug(f"Debugging enabled on {package_name!r}")
|
proxy_reader/protocols/reader.py
CHANGED
|
@@ -5,102 +5,70 @@ from ..proxy import Proxy
|
|
|
5
5
|
|
|
6
6
|
class ProxiesReaderProtocol(Protocol):
|
|
7
7
|
@property
|
|
8
|
-
def total(self) -> int:
|
|
9
|
-
pass
|
|
8
|
+
def total(self) -> int: ...
|
|
10
9
|
|
|
11
10
|
@property
|
|
12
|
-
def total_working(self) -> int:
|
|
13
|
-
pass
|
|
11
|
+
def total_working(self) -> int: ...
|
|
14
12
|
|
|
15
13
|
@property
|
|
16
|
-
def total_bad(self) -> int:
|
|
17
|
-
pass
|
|
14
|
+
def total_bad(self) -> int: ...
|
|
18
15
|
|
|
19
16
|
@property
|
|
20
|
-
def proxies(self) -> ProxiesList:
|
|
21
|
-
pass
|
|
17
|
+
def proxies(self) -> ProxiesList: ...
|
|
22
18
|
|
|
23
19
|
@property
|
|
24
|
-
def bad_proxies(self) -> ProxiesList:
|
|
25
|
-
pass
|
|
20
|
+
def bad_proxies(self) -> ProxiesList: ...
|
|
26
21
|
|
|
27
22
|
@property
|
|
28
|
-
def working_proxies(self) -> ProxiesList:
|
|
29
|
-
pass
|
|
23
|
+
def working_proxies(self) -> ProxiesList: ...
|
|
30
24
|
|
|
31
25
|
@working_proxies.setter
|
|
32
|
-
def working_proxies(self, working_proxies: List[Proxy]) -> None:
|
|
33
|
-
pass
|
|
26
|
+
def working_proxies(self, working_proxies: List[Proxy]) -> None: ...
|
|
34
27
|
|
|
35
|
-
def
|
|
36
|
-
pass
|
|
28
|
+
def _random_proxy_check_url(self) -> str: ...
|
|
37
29
|
|
|
38
|
-
def
|
|
39
|
-
pass
|
|
30
|
+
def read_with_auth(self) -> None: ...
|
|
40
31
|
|
|
41
|
-
def
|
|
42
|
-
pass
|
|
43
|
-
|
|
44
|
-
def read_authless(self) -> None:
|
|
45
|
-
pass
|
|
32
|
+
def read_authless(self) -> None: ...
|
|
46
33
|
|
|
47
34
|
async def _check_proxy(
|
|
48
35
|
self, proxy: Proxy, response_time: Optional[int] = None
|
|
49
|
-
) -> bool:
|
|
50
|
-
pass
|
|
36
|
+
) -> bool: ...
|
|
51
37
|
|
|
52
|
-
async def check_all_proxies(self, max_resp_time: int = 30) -> None:
|
|
53
|
-
pass
|
|
38
|
+
async def check_all_proxies(self, max_resp_time: int = 30) -> None: ...
|
|
54
39
|
|
|
55
40
|
async def _check_proxy_socks(
|
|
56
41
|
self, proxy: Proxy, response_time: Optional[int] = None
|
|
57
|
-
) -> bool:
|
|
58
|
-
pass
|
|
42
|
+
) -> bool: ...
|
|
59
43
|
|
|
60
|
-
async def check_all_proxies_socks5(self, max_resp_time: int = 5) -> None:
|
|
61
|
-
pass
|
|
44
|
+
async def check_all_proxies_socks5(self, max_resp_time: int = 5) -> None: ...
|
|
62
45
|
|
|
63
|
-
def get_working_proxies_list_http(self) -> List[str]:
|
|
64
|
-
pass
|
|
46
|
+
def get_working_proxies_list_http(self) -> List[str]: ...
|
|
65
47
|
|
|
66
|
-
def write_working_proxies(self, filename: str) -> None:
|
|
67
|
-
pass
|
|
48
|
+
def write_working_proxies(self, filename: str) -> None: ...
|
|
68
49
|
|
|
69
|
-
def get_random_http(self) -> Optional[str]:
|
|
70
|
-
pass
|
|
50
|
+
def get_random_http(self) -> Optional[str]: ...
|
|
71
51
|
|
|
72
|
-
def get_random_socks5(self) -> Optional[str]:
|
|
73
|
-
pass
|
|
52
|
+
def get_random_socks5(self) -> Optional[str]: ...
|
|
74
53
|
|
|
75
|
-
def get_random_socks5_telegram(self) -> Optional[Dict[str, Any]]:
|
|
76
|
-
pass
|
|
54
|
+
def get_random_socks5_telegram(self) -> Optional[Dict[str, Any]]: ...
|
|
77
55
|
|
|
78
|
-
def next_http_from_list(self) -> Optional[str]:
|
|
79
|
-
pass
|
|
56
|
+
def next_http_from_list(self) -> Optional[str]: ...
|
|
80
57
|
|
|
81
|
-
def next_http_from_cycle(self) -> str:
|
|
82
|
-
pass
|
|
58
|
+
def next_http_from_cycle(self) -> str: ...
|
|
83
59
|
|
|
84
|
-
def next_socks5_from_list(self) -> str:
|
|
85
|
-
pass
|
|
60
|
+
def next_socks5_from_list(self) -> str: ...
|
|
86
61
|
|
|
87
|
-
def next_socks5_from_cycle(self) -> str:
|
|
88
|
-
pass
|
|
62
|
+
def next_socks5_from_cycle(self) -> str: ...
|
|
89
63
|
|
|
90
|
-
def next_http_telegram_from_list(self) -> Dict[str, Any]:
|
|
91
|
-
pass
|
|
64
|
+
def next_http_telegram_from_list(self) -> Dict[str, Any]: ...
|
|
92
65
|
|
|
93
|
-
def next_http_telegram_from_cycle(self) -> Dict[str, Any]:
|
|
94
|
-
pass
|
|
66
|
+
def next_http_telegram_from_cycle(self) -> Dict[str, Any]: ...
|
|
95
67
|
|
|
96
|
-
def next_socks5_telegram_from_cycle(self) -> Dict[str, Any]:
|
|
97
|
-
pass
|
|
68
|
+
def next_socks5_telegram_from_cycle(self) -> Dict[str, Any]: ...
|
|
98
69
|
|
|
99
|
-
def next_socks5_telegram_from_list(self) -> Dict[str, Any]:
|
|
100
|
-
pass
|
|
70
|
+
def next_socks5_telegram_from_list(self) -> Dict[str, Any]: ...
|
|
101
71
|
|
|
102
|
-
def next_https_from_list(self) -> str:
|
|
103
|
-
pass
|
|
72
|
+
def next_https_from_list(self) -> str: ...
|
|
104
73
|
|
|
105
|
-
def next_https_from_cycle(self) -> str:
|
|
106
|
-
pass
|
|
74
|
+
def next_https_from_cycle(self) -> str: ...
|
proxy_reader/proxy.py
CHANGED
|
@@ -1,78 +1,83 @@
|
|
|
1
|
-
from
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any, Dict, Optional
|
|
4
|
+
|
|
5
|
+
if TYPE_CHECKING:
|
|
6
|
+
from ._types import GeneralDict, ProxyDictT
|
|
2
7
|
|
|
3
8
|
|
|
4
9
|
class Proxy:
|
|
5
|
-
def __init__(
|
|
6
|
-
self
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
self.
|
|
10
|
+
def __init__(
|
|
11
|
+
self,
|
|
12
|
+
proxy: ProxyDictT,
|
|
13
|
+
) -> None:
|
|
14
|
+
self._proxy = proxy
|
|
10
15
|
|
|
11
16
|
@property
|
|
12
17
|
def ip(self) -> str:
|
|
13
|
-
return self.
|
|
18
|
+
return self._proxy["host"]
|
|
14
19
|
|
|
15
20
|
@property
|
|
16
21
|
def port(self) -> str:
|
|
17
|
-
return self.
|
|
22
|
+
return self._proxy["port"]
|
|
18
23
|
|
|
19
24
|
@property
|
|
20
25
|
def username(self) -> Optional[str]:
|
|
21
|
-
return self.
|
|
26
|
+
return self._proxy["username"]
|
|
22
27
|
|
|
23
28
|
@property
|
|
24
29
|
def password(self) -> Optional[str]:
|
|
25
|
-
return self.
|
|
30
|
+
return self._proxy["password"]
|
|
26
31
|
|
|
27
32
|
@property
|
|
28
33
|
def http(self) -> str:
|
|
29
34
|
if self.username and self.password:
|
|
30
|
-
return f"http://{self.
|
|
31
|
-
return f"http://{self.
|
|
35
|
+
return f"http://{self.username}:{self.password}@{self.ip}:{self.port}"
|
|
36
|
+
return f"http://{self.ip}:{self.port}"
|
|
32
37
|
|
|
33
38
|
@property
|
|
34
39
|
def https(self) -> str:
|
|
35
40
|
if self.username and self.password:
|
|
36
|
-
return f"https://{self.
|
|
37
|
-
return f"https://{self.
|
|
41
|
+
return f"https://{self.username}:{self.password}@{self.ip}:{self.port}"
|
|
42
|
+
return f"https://{self.ip}:{self.port}"
|
|
38
43
|
|
|
39
44
|
@property
|
|
40
45
|
def telegram_http(self) -> Dict[str, Any]:
|
|
41
|
-
p = {"proxy_type": 3, "addr": self.
|
|
46
|
+
p: GeneralDict = {"proxy_type": 3, "addr": self.ip, "port": int(self.port)}
|
|
42
47
|
if self.username and self.password:
|
|
43
48
|
p.update({"username": self.username, "password": self.password})
|
|
44
49
|
return p
|
|
45
50
|
|
|
46
51
|
@property
|
|
47
52
|
def telegram_socks5(self) -> Dict[str, Any]:
|
|
48
|
-
p = {"proxy_type": 2, "addr": self.
|
|
53
|
+
p: GeneralDict = {"proxy_type": 2, "addr": self.ip, "port": int(self.port)}
|
|
49
54
|
if self.username and self.password:
|
|
50
55
|
p.update({"username": self.username, "password": self.password})
|
|
51
56
|
return p
|
|
52
57
|
|
|
53
58
|
@property
|
|
54
59
|
def telegram_socks4(self) -> Dict[str, Any]:
|
|
55
|
-
p = {"proxy_type": 1, "addr": self.
|
|
60
|
+
p: GeneralDict = {"proxy_type": 1, "addr": self.ip, "port": int(self.port)}
|
|
56
61
|
if self.username and self.password:
|
|
57
62
|
p.update({"username": self.username, "password": self.password})
|
|
58
63
|
return p
|
|
59
64
|
|
|
60
65
|
@property
|
|
61
66
|
def socks5(self) -> str:
|
|
62
|
-
if self.
|
|
63
|
-
return f"socks5://{self.
|
|
67
|
+
if self.username is not None and self.password is not None:
|
|
68
|
+
return f"socks5://{self.username}:{self.password}@{self.ip}:{self.port}"
|
|
64
69
|
return f"socks5://{self.ip}:{self.port}"
|
|
65
70
|
|
|
66
71
|
@property
|
|
67
72
|
def socks4(self) -> str:
|
|
68
|
-
if self.
|
|
69
|
-
return f"socks4://{self.
|
|
73
|
+
if self.username is not None and self.password is not None:
|
|
74
|
+
return f"socks4://{self.username}:{self.password}@{self.ip}:{self.port}"
|
|
70
75
|
return f"socks4://{self.ip}:{self.port}"
|
|
71
76
|
|
|
72
77
|
def __str__(self) -> str:
|
|
73
|
-
if self.
|
|
74
|
-
return f"{self.
|
|
75
|
-
return f"{self.
|
|
78
|
+
if self.username is not None and self.password is not None:
|
|
79
|
+
return f"{self.ip}:{self.port}:{self.username}:{self.password}"
|
|
80
|
+
return f"{self.ip}:{self.port}"
|
|
76
81
|
|
|
77
82
|
def __repr__(self) -> str:
|
|
78
83
|
return self.__str__()
|
proxy_reader/reader.py
CHANGED
|
@@ -1,40 +1,54 @@
|
|
|
1
|
-
import random
|
|
2
|
-
import itertools
|
|
3
|
-
from .proxy import Proxy
|
|
4
|
-
import aiohttp
|
|
5
1
|
import asyncio
|
|
6
|
-
|
|
2
|
+
import itertools
|
|
7
3
|
import os
|
|
8
|
-
|
|
4
|
+
import random
|
|
9
5
|
import sys
|
|
10
|
-
|
|
11
|
-
|
|
6
|
+
import tempfile
|
|
7
|
+
import warnings
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import aiohttp
|
|
11
|
+
from aiohttp_socks import ProxyConnector
|
|
12
|
+
|
|
13
|
+
from ._types import ProxiesList, ProxyDictT, ProxyiesGen
|
|
14
|
+
from .logs_config import logger
|
|
12
15
|
from .protocols.reader import ProxiesReaderProtocol
|
|
16
|
+
from .proxy import Proxy
|
|
17
|
+
from .utils import parse_proxy_line
|
|
13
18
|
|
|
14
19
|
|
|
15
20
|
class ProxiesReader(ProxiesReaderProtocol):
|
|
16
21
|
def __init__(
|
|
17
22
|
self,
|
|
18
|
-
|
|
23
|
+
proxies_file: str,
|
|
24
|
+
check_proxies: bool = False,
|
|
25
|
+
proxy_checking_threads: int = 50,
|
|
26
|
+
max_response_time: int = 60,
|
|
19
27
|
shuffle: bool = False,
|
|
20
|
-
debug: bool = False,
|
|
21
|
-
extra_debug: bool = False,
|
|
22
28
|
) -> None:
|
|
23
|
-
self.
|
|
24
|
-
self._debug = debug
|
|
25
|
-
self._extra_debug = extra_debug
|
|
29
|
+
self._raw_proxies = open(proxies_file, encoding="utf-8", mode="r").readlines()
|
|
26
30
|
|
|
31
|
+
self._check_proxies = check_proxies
|
|
27
32
|
self._shuffle = shuffle
|
|
28
|
-
self.
|
|
29
|
-
|
|
33
|
+
self._proxies_dict_list: list[ProxyDictT] = [
|
|
34
|
+
parse_proxy_line(p) for p in self._raw_proxies
|
|
35
|
+
]
|
|
36
|
+
self._all_proxies: ProxiesList = [Proxy(p) for p in self._proxies_dict_list]
|
|
37
|
+
|
|
30
38
|
self._bad_proxies: ProxiesList = []
|
|
31
|
-
|
|
32
|
-
self.
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
self.
|
|
39
|
+
|
|
40
|
+
self._working_proxies: ProxiesList = (
|
|
41
|
+
[] if self._check_proxies else self._all_proxies
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
self._proxy_iterator: ProxyiesGen | None = None
|
|
45
|
+
self._proxy_iterator_cycle: ProxyiesGen | None = None
|
|
46
|
+
self._thread_control: asyncio.Semaphore = asyncio.Semaphore(
|
|
47
|
+
proxy_checking_threads
|
|
48
|
+
)
|
|
49
|
+
self._max_response_time = max_response_time
|
|
37
50
|
self._timeout_count = 0
|
|
51
|
+
self._proxies_checked = False
|
|
38
52
|
|
|
39
53
|
self._check_urls = [
|
|
40
54
|
# New
|
|
@@ -53,17 +67,39 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
53
67
|
# "http://dog.ceo/api/breeds/image/random",
|
|
54
68
|
]
|
|
55
69
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
70
|
+
@classmethod
|
|
71
|
+
def load_list(
|
|
72
|
+
cls,
|
|
73
|
+
proxies: list[str],
|
|
74
|
+
check_proxies: bool = False,
|
|
75
|
+
proxy_checking_threads: int = 50,
|
|
76
|
+
max_response_time: int = 60,
|
|
77
|
+
shuffle: bool = False,
|
|
78
|
+
) -> "ProxiesReader":
|
|
79
|
+
"""Load proxies from a list"""
|
|
80
|
+
proxies_file = tempfile.NamedTemporaryFile(
|
|
81
|
+
delete=False, suffix=".txt", mode="w+"
|
|
82
|
+
)
|
|
83
|
+
try:
|
|
84
|
+
proxies_file.write("\n".join(proxies))
|
|
85
|
+
proxies_file.flush()
|
|
86
|
+
new = cls(
|
|
87
|
+
proxies_file.name,
|
|
88
|
+
check_proxies,
|
|
89
|
+
proxy_checking_threads,
|
|
90
|
+
max_response_time,
|
|
91
|
+
shuffle,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
finally:
|
|
95
|
+
proxies_file.close()
|
|
96
|
+
os.unlink(proxies_file.name)
|
|
97
|
+
|
|
98
|
+
return new
|
|
63
99
|
|
|
64
100
|
@property
|
|
65
101
|
def total(self) -> int:
|
|
66
|
-
return len(self.
|
|
102
|
+
return len(self._all_proxies)
|
|
67
103
|
|
|
68
104
|
@property
|
|
69
105
|
def total_working(self) -> int:
|
|
@@ -75,7 +111,7 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
75
111
|
|
|
76
112
|
@property
|
|
77
113
|
def proxies(self) -> ProxiesList:
|
|
78
|
-
return self.
|
|
114
|
+
return self._all_proxies
|
|
79
115
|
|
|
80
116
|
@property
|
|
81
117
|
def bad_proxies(self) -> ProxiesList:
|
|
@@ -90,54 +126,56 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
90
126
|
self._working_proxies = working_proxies
|
|
91
127
|
|
|
92
128
|
def __str__(self) -> str:
|
|
93
|
-
return str(self.
|
|
129
|
+
return str(self._all_proxies)
|
|
94
130
|
|
|
95
131
|
def __repr__(self) -> str:
|
|
96
132
|
return self.__str__()
|
|
97
133
|
|
|
98
|
-
def
|
|
99
|
-
lines = open(self._file_path).readlines()
|
|
100
|
-
return [line.strip().replace("\n", "") for line in lines]
|
|
101
|
-
|
|
102
|
-
def random_url(self) -> str:
|
|
134
|
+
def _random_proxy_check_url(self) -> str:
|
|
103
135
|
return random.choice(self._check_urls)
|
|
104
136
|
|
|
105
137
|
def read_with_auth(self) -> None:
|
|
106
138
|
"""Format: IP:PORT:USERNAME:PASSWORD"""
|
|
107
|
-
|
|
139
|
+
return warnings.warn(
|
|
140
|
+
"read_with_auth is deprecated. Please don't use it. It's not working ...",
|
|
141
|
+
)
|
|
142
|
+
raw_proxies = self._read_raw()
|
|
108
143
|
for proxy in raw_proxies:
|
|
109
144
|
sp_proxy = proxy.split(":")
|
|
110
145
|
ip = sp_proxy[0]
|
|
111
146
|
port = sp_proxy[1]
|
|
112
147
|
username = sp_proxy[2]
|
|
113
148
|
password = sp_proxy[3]
|
|
114
|
-
self.
|
|
149
|
+
self._all_proxies.append(Proxy(ip, port, username, password))
|
|
115
150
|
|
|
116
151
|
self._has_auth = True
|
|
117
152
|
if self._shuffle:
|
|
118
|
-
random.shuffle(self.
|
|
153
|
+
random.shuffle(self._all_proxies)
|
|
119
154
|
|
|
120
155
|
def read_authless(self) -> None:
|
|
121
156
|
"""Format: IP:PORT"""
|
|
122
|
-
|
|
157
|
+
return warnings.warn(
|
|
158
|
+
"read_authless is deprecated. Please don't use it. It's not working ..."
|
|
159
|
+
)
|
|
160
|
+
raw_proxies = self._read_raw()
|
|
123
161
|
for proxy in raw_proxies:
|
|
124
162
|
sp_proxy = proxy.split(":")
|
|
125
163
|
ip = sp_proxy[0]
|
|
126
164
|
port = sp_proxy[1]
|
|
127
|
-
self.
|
|
165
|
+
self._all_proxies.append(Proxy(ip, port))
|
|
128
166
|
logger.debug(
|
|
129
|
-
f"Loaded total {len(self.
|
|
167
|
+
f"Loaded total {len(self._all_proxies)} proxies from {self._file_path}"
|
|
130
168
|
)
|
|
131
169
|
if self._shuffle:
|
|
132
|
-
random.shuffle(self.
|
|
170
|
+
random.shuffle(self._all_proxies)
|
|
133
171
|
|
|
134
172
|
async def _check_proxy(
|
|
135
|
-
self, proxy: Proxy, response_time:
|
|
173
|
+
self, proxy: Proxy, response_time: int | None = None
|
|
136
174
|
) -> bool:
|
|
137
175
|
connectins_limit = 60 if "win" in sys.platform else 100
|
|
138
176
|
connector = aiohttp.TCPConnector(limit=connectins_limit)
|
|
139
177
|
session = aiohttp.ClientSession(connector=connector)
|
|
140
|
-
url = self.
|
|
178
|
+
url = self._random_proxy_check_url()
|
|
141
179
|
p = proxy.http
|
|
142
180
|
|
|
143
181
|
async with self._thread_control:
|
|
@@ -156,14 +194,17 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
156
194
|
self._timeout_count += 1
|
|
157
195
|
logger.debug(f"{p} : TIMEOUT {e}. {url}")
|
|
158
196
|
self._bad_proxies.append(proxy)
|
|
197
|
+
await connector.close()
|
|
159
198
|
await session.close()
|
|
160
199
|
return False
|
|
161
200
|
|
|
162
201
|
except Exception as e:
|
|
163
|
-
logger.debug(f"Bad proxy raised. {e}", exc_info=
|
|
164
|
-
await session.close()
|
|
202
|
+
logger.debug(f"Bad proxy raised. {e}", exc_info=True)
|
|
165
203
|
return False
|
|
166
204
|
|
|
205
|
+
finally:
|
|
206
|
+
await session.close()
|
|
207
|
+
|
|
167
208
|
if resp.status == 200:
|
|
168
209
|
logger.debug(f"{p}: Working")
|
|
169
210
|
self._working_proxies.append(proxy)
|
|
@@ -176,19 +217,19 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
176
217
|
|
|
177
218
|
async def check_all_proxies(self, max_resp_time: int = 30) -> None:
|
|
178
219
|
"""Run this to check all proxies at once."""
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
220
|
+
if self._check_proxies:
|
|
221
|
+
async with asyncio.TaskGroup() as gp:
|
|
222
|
+
for proxy in self._all_proxies:
|
|
223
|
+
gp.create_task(self._check_proxy(proxy, max_resp_time))
|
|
224
|
+
self._proxies_checked = True
|
|
225
|
+
logger.debug("All proxies checked.")
|
|
185
226
|
|
|
186
227
|
async def _check_proxy_socks(
|
|
187
|
-
self, proxy: Proxy, response_time:
|
|
228
|
+
self, proxy: Proxy, response_time: int | None = None
|
|
188
229
|
) -> bool:
|
|
189
|
-
url = self.
|
|
230
|
+
url = self._random_proxy_check_url()
|
|
190
231
|
socks_connector = ProxyConnector.from_url(proxy.socks5) # type: ignore
|
|
191
|
-
session = aiohttp.ClientSession(connector=socks_connector)
|
|
232
|
+
session = aiohttp.ClientSession(connector=socks_connector) # type: ignore
|
|
192
233
|
logger.debug(f"Checking proxy {proxy} ..")
|
|
193
234
|
try:
|
|
194
235
|
resp = await asyncio.wait_for(session.get(url), timeout=response_time)
|
|
@@ -200,7 +241,7 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
200
241
|
return False
|
|
201
242
|
|
|
202
243
|
except Exception as e:
|
|
203
|
-
logger.debug(f"Bad proxy raised. {e}", exc_info=
|
|
244
|
+
logger.debug(f"Bad proxy raised. {e}", exc_info=True)
|
|
204
245
|
await session.close()
|
|
205
246
|
return False
|
|
206
247
|
|
|
@@ -217,8 +258,8 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
217
258
|
|
|
218
259
|
async def check_all_proxies_socks5(self, max_resp_time: int = 5) -> None:
|
|
219
260
|
"""Run the check on all proxies at once."""
|
|
220
|
-
tasks:
|
|
221
|
-
for proxy in self.
|
|
261
|
+
tasks: list[asyncio.Task[bool]] = []
|
|
262
|
+
for proxy in self._all_proxies:
|
|
222
263
|
tasks.append(
|
|
223
264
|
asyncio.create_task(self._check_proxy_socks(proxy, max_resp_time))
|
|
224
265
|
)
|
|
@@ -226,8 +267,8 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
226
267
|
self._proxies_checked = True
|
|
227
268
|
logger.debug("All proxies checked.")
|
|
228
269
|
|
|
229
|
-
def get_working_proxies_list_http(self) ->
|
|
230
|
-
working_list:
|
|
270
|
+
def get_working_proxies_list_http(self) -> list[str]:
|
|
271
|
+
working_list: list[str] = []
|
|
231
272
|
for proxy in self._working_proxies:
|
|
232
273
|
working_list.append(proxy.http)
|
|
233
274
|
return working_list
|
|
@@ -239,28 +280,28 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
239
280
|
f.write("\n".join([proxy.strip() for proxy in working_list]))
|
|
240
281
|
logger.debug(f"Proxies written to: {filename}")
|
|
241
282
|
|
|
242
|
-
def get_random_http(self) ->
|
|
283
|
+
def get_random_http(self) -> str | None:
|
|
243
284
|
_p = None
|
|
244
285
|
if len(self._working_proxies) > 0:
|
|
245
286
|
proxy = random.choice(self._working_proxies)
|
|
246
287
|
_p = proxy.http
|
|
247
288
|
return _p
|
|
248
289
|
|
|
249
|
-
def get_random_socks5(self) ->
|
|
290
|
+
def get_random_socks5(self) -> str | None:
|
|
250
291
|
_p = None
|
|
251
292
|
if len(self._working_proxies) > 0:
|
|
252
293
|
proxy = random.choice(self._working_proxies)
|
|
253
294
|
_p = proxy.socks5
|
|
254
295
|
return _p
|
|
255
296
|
|
|
256
|
-
def get_random_socks5_telegram(self) ->
|
|
297
|
+
def get_random_socks5_telegram(self) -> dict[str, Any] | None:
|
|
257
298
|
_p = None
|
|
258
299
|
if len(self._working_proxies) > 0:
|
|
259
300
|
proxy = random.choice(self._working_proxies)
|
|
260
301
|
_p = proxy.telegram_socks5
|
|
261
302
|
return _p
|
|
262
303
|
|
|
263
|
-
def next_http_from_list(self) ->
|
|
304
|
+
def next_http_from_list(self) -> str | None:
|
|
264
305
|
"""Get next proxy from proxies list"""
|
|
265
306
|
|
|
266
307
|
def __iter() -> ProxyiesGen:
|
|
@@ -300,14 +341,14 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
300
341
|
)
|
|
301
342
|
return str(next(self._proxy_iterator_cycle).socks5)
|
|
302
343
|
|
|
303
|
-
def next_http_telegram_from_list(self) ->
|
|
344
|
+
def next_http_telegram_from_list(self) -> dict[str, Any]:
|
|
304
345
|
"""Get next proxy from proxies list"""
|
|
305
346
|
|
|
306
347
|
if self._proxy_iterator is None:
|
|
307
348
|
self._proxy_iterator = self.__proxies_gen(self._working_proxies)
|
|
308
349
|
return dict(next(self._proxy_iterator).telegram_http)
|
|
309
350
|
|
|
310
|
-
def next_http_telegram_from_cycle(self) ->
|
|
351
|
+
def next_http_telegram_from_cycle(self) -> dict[str, Any]:
|
|
311
352
|
"""Get next proxy from proxies cycle"""
|
|
312
353
|
|
|
313
354
|
if self._proxy_iterator is None:
|
|
@@ -316,7 +357,7 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
316
357
|
)
|
|
317
358
|
return dict(next(self._proxy_iterator).telegram_http)
|
|
318
359
|
|
|
319
|
-
def next_socks5_telegram_from_cycle(self) ->
|
|
360
|
+
def next_socks5_telegram_from_cycle(self) -> dict[str, Any]:
|
|
320
361
|
"""Get next proxy from proxies cycle"""
|
|
321
362
|
if self._proxy_iterator_cycle is None:
|
|
322
363
|
self._proxy_iterator_cycle = self.__proxies_gen(
|
|
@@ -324,7 +365,7 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
324
365
|
)
|
|
325
366
|
return dict(next(self._proxy_iterator_cycle).telegram_socks5)
|
|
326
367
|
|
|
327
|
-
def next_socks5_telegram_from_list(self) ->
|
|
368
|
+
def next_socks5_telegram_from_list(self) -> dict[str, Any]:
|
|
328
369
|
"""Get next proxy from proxies cycle"""
|
|
329
370
|
|
|
330
371
|
if self._proxy_iterator_cycle is None:
|
|
@@ -350,7 +391,7 @@ class ProxiesReader(ProxiesReaderProtocol):
|
|
|
350
391
|
return str(next(self._proxy_iterator_cycle).https)
|
|
351
392
|
|
|
352
393
|
def __proxies_gen(
|
|
353
|
-
self, proxies_list:
|
|
394
|
+
self, proxies_list: list[Proxy], cycle_proxies: bool = False
|
|
354
395
|
) -> ProxyiesGen:
|
|
355
396
|
if cycle_proxies:
|
|
356
397
|
for proxy in itertools.cycle(self._working_proxies):
|
proxy_reader/utils.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from typing import cast
|
|
3
|
+
|
|
4
|
+
from ._types import ProxyDictT
|
|
5
|
+
from .logs_config import logger
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def parse_proxy_line(proxy: str) -> ProxyDictT:
|
|
9
|
+
"""
|
|
10
|
+
Detects the format of the proxy string and extracts the components.
|
|
11
|
+
|
|
12
|
+
Returns a dictionary with 'host', 'port', 'username', and 'password'.
|
|
13
|
+
"""
|
|
14
|
+
logger.info("Reading the proxy format ...")
|
|
15
|
+
patterns: list[str] = [
|
|
16
|
+
# Format 1: IP/Hostname:PORT:USERNAME:PASSWORD
|
|
17
|
+
r"^(?P<host>[\w\.-]+):(?P<port>\d+):(?P<username>[^:]+):(?P<password>.+)$",
|
|
18
|
+
# Format 2: USERNAME:PASSWORD:IP/Hostname:PORT
|
|
19
|
+
r"^(?P<username>[^:]+):(?P<password>[^:]+):(?P<host>[\w\.-]+):(?P<port>\d+)$",
|
|
20
|
+
# Format 3: http://USERNAME:PASSWORD@IP/Hostname:PORT
|
|
21
|
+
r"^http:\/\/(?P<username>[^:]+):(?P<password>[^@]+)@(?P<host>[\w\.-]+):(?P<port>\d+)$",
|
|
22
|
+
# Format 4: USERNAME:PASSWORD@IP/Hostname:PORT
|
|
23
|
+
r"^(?P<username>[^:]+):(?P<password>[^@]+)@(?P<host>[\w\.-]+):(?P<port>\d+)$",
|
|
24
|
+
# Format 5: IP/Hostname:PORT (No Username/Password)
|
|
25
|
+
r"^(?P<host>[\w\.-]+):(?P<port>\d+)$",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
for pattern in patterns:
|
|
29
|
+
match = re.match(pattern, proxy.strip())
|
|
30
|
+
if match:
|
|
31
|
+
data = match.groupdict()
|
|
32
|
+
data.setdefault("username", None) # Ensure username exists
|
|
33
|
+
data.setdefault("password", None) # Ensure password exists
|
|
34
|
+
return cast(ProxyDictT, data)
|
|
35
|
+
|
|
36
|
+
raise ValueError(f"Invalid format: {proxy}")
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: proxy-reader
|
|
3
|
+
Version: 2.1.0
|
|
4
|
+
Summary: Proxy reader for Python
|
|
5
|
+
Project-URL: Homepage, https://github.com/runetech0/proxy-reader
|
|
6
|
+
Project-URL: github, https://github.com/runetech0/proxy-reader
|
|
7
|
+
Requires-Python: >=3.12
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: aiohttp>=3.12.15
|
|
11
|
+
Requires-Dist: aiohttp-socks>=0.10.1
|
|
12
|
+
Requires-Dist: pytest>=8.4.1
|
|
13
|
+
Provides-Extra: test
|
|
14
|
+
Requires-Dist: pytest>=7.0.0; extra == "test"
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# proxy-reader - A simple but useful bulk proxies reader and checker.
|
|
21
|
+
|
|
22
|
+
This is useful when you are working with multiple proxies and want to bulk check
|
|
23
|
+
these proxies before using them.
|
|
24
|
+
It has iterators to iterate the proxies for easy reading.
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
## Installation
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
pip install proxy-reader
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Changelogs
|
|
34
|
+
|
|
35
|
+
### [v0.3.0]
|
|
36
|
+
* Add load_list class method to load proxies from list
|
|
37
|
+
* MAJOR: auto detect the proxies format
|
|
38
|
+
* Use new logging config to disable logs and prevent proxy_reader.log from creating
|
|
39
|
+
* Logs can be enabled from user-side
|
|
40
|
+
* Add option to disable proxies checking from __init__
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
### [v0.2.0] - 2024-10-21
|
|
44
|
+
|
|
45
|
+
* Previous unrecorded changes
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
For more info/queries Telegram: [@runetech](https://t.me/runetech)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
proxy_reader/__init__.py,sha256=L3-vb_-fS3dfrnunLqTxpqwIOq_ygiV8cKZpfJsXWns,45
|
|
2
|
+
proxy_reader/_types.py,sha256=C285ga4m1KGJ9IREBGzXp6bdOIOm87rJKu2USQ1tZG0,488
|
|
3
|
+
proxy_reader/domains.py,sha256=d4FUb2zw3Kg9bAjtaggEcxZA293WvVcRTgMKKPmYr1A,135662
|
|
4
|
+
proxy_reader/logs_config.py,sha256=SXNDi0op7Bk69qxbVsYajdSAfhhw9eTVVdf6MS2v6Ts,747
|
|
5
|
+
proxy_reader/proxy.py,sha256=THieTNRtZEl86ijK5nFbSqQ030alzkFFU7aSkOPr6JU,2676
|
|
6
|
+
proxy_reader/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
proxy_reader/reader.py,sha256=vlehkoWcgO3QoWBV2X33_HUzCzphfIogaH-Autl-USk,13519
|
|
8
|
+
proxy_reader/utils.py,sha256=LkRUug8Utm-zsje50L3n80V7DPJ8FSHjoyQKEPlUIOY,1451
|
|
9
|
+
proxy_reader/protocols/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
+
proxy_reader/protocols/reader.py,sha256=UbQKOPhIqRwaGpgN29-zdUs1hYOaBKCyqjgb9gQHrD0,2004
|
|
11
|
+
proxy_reader-2.1.0.dist-info/licenses/LICENSE,sha256=8DyHi9RsZXeM8s1DpaXubZSSG44PkzV2oskJE3E09F4,1081
|
|
12
|
+
proxy_reader-2.1.0.dist-info/METADATA,sha256=gWBCI39LWS5UdC08lSQ8vmyAK0ExnkAJDv_PWNbplDQ,1229
|
|
13
|
+
proxy_reader-2.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
14
|
+
proxy_reader-2.1.0.dist-info/top_level.txt,sha256=NIrKvLf5DF2_oEnj0yg84roFna2-MaJ7zE1jY2CIU0I,13
|
|
15
|
+
proxy_reader-2.1.0.dist-info/RECORD,,
|
|
File without changes
|
proxy_reader/logger.py
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
import logging
|
|
4
|
-
from logging.handlers import RotatingFileHandler
|
|
5
|
-
|
|
6
|
-
filename = "proxy_reader.log"
|
|
7
|
-
logging_format = logging.Formatter(
|
|
8
|
-
"%(levelname)s:[%(filename)s:%(lineno)s]:%(asctime)s: %(message)s")
|
|
9
|
-
logger = logging.getLogger(__name__)
|
|
10
|
-
console_handler = logging.StreamHandler()
|
|
11
|
-
console_handler.setFormatter(logging_format)
|
|
12
|
-
file_handler = RotatingFileHandler(filename, mode='a', maxBytes=5 * 1024 * 1024,
|
|
13
|
-
backupCount=2, encoding="utf-8", delay=False)
|
|
14
|
-
file_handler.setFormatter(logging_format)
|
|
15
|
-
|
|
16
|
-
logger.setLevel(logging.DEBUG)
|
|
17
|
-
file_handler.setLevel(logging.DEBUG)
|
|
18
|
-
console_handler.setLevel(logging.DEBUG)
|
|
19
|
-
|
|
20
|
-
logger.addHandler(console_handler)
|
|
21
|
-
logger.addHandler(file_handler)
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: proxy-reader
|
|
3
|
-
Version: 0.1.0
|
|
4
|
-
Summary: Read and check bulk proxies effectively
|
|
5
|
-
Author-email: Rune Tech <runetech2024@gmail.com>
|
|
6
|
-
License: The MIT License (MIT)
|
|
7
|
-
Copyright © 2024 Rune Tech
|
|
8
|
-
|
|
9
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
10
|
-
|
|
11
|
-
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
12
|
-
|
|
13
|
-
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
14
|
-
Project-URL: Homepage, https://github.com/runetech0/proxy-reader
|
|
15
|
-
Project-URL: github, https://github.com/runetech0/proxy-reader
|
|
16
|
-
Keywords: proxies,proxy,reader,checker,bulk
|
|
17
|
-
Requires-Python: >=3.11
|
|
18
|
-
Description-Content-Type: text/markdown
|
|
19
|
-
License-File: LICENSE
|
|
20
|
-
Requires-Dist: aiohttp >=3.10.10
|
|
21
|
-
Requires-Dist: aiohttp-socks >=0.9.0
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
# proxy_reader - A simple but useful bulk proxies reader and checker.
|
|
27
|
-
|
|
28
|
-
This is useful when you are working with multiple proxies and want to bulk check
|
|
29
|
-
these proxies before using them.
|
|
30
|
-
It has iterators to iterate the proxies for easy reading.
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
## Changelogs
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
proxy_reader/__init__.py,sha256=8_HgszorTyU_0OJmvxgimP9S21y07s9R1n_U-wHMwTs,46
|
|
2
|
-
proxy_reader/_types.py,sha256=OWAYiN91E_IWgs1myPy1Q9l-8XkVVq8uDzjLOTB_Eww,317
|
|
3
|
-
proxy_reader/domains.py,sha256=d4FUb2zw3Kg9bAjtaggEcxZA293WvVcRTgMKKPmYr1A,135662
|
|
4
|
-
proxy_reader/logger.py,sha256=BxaHptwZlPPGdx5mbS8vi77u0N-W6cKhcR-5lG2x93c,710
|
|
5
|
-
proxy_reader/proxy.py,sha256=vkI2vonxlm0x0u3Tojq6xKaw5_7fvSbmICW8b_M4lzY,2645
|
|
6
|
-
proxy_reader/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
-
proxy_reader/reader.py,sha256=KeaAPhxMI6moYaLmzYj7ZGcn8REXWs-1xBMkXuK2Ff0,12373
|
|
8
|
-
proxy_reader/protocols/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
-
proxy_reader/protocols/reader.py,sha256=WscEXPpKmJuMF2R7EBqDZTheRh8BIUilukojcPRFyRg,2303
|
|
10
|
-
proxy_reader-0.1.0.dist-info/LICENSE,sha256=8DyHi9RsZXeM8s1DpaXubZSSG44PkzV2oskJE3E09F4,1081
|
|
11
|
-
proxy_reader-0.1.0.dist-info/METADATA,sha256=hsWGBfcZrYHMTZWKY_oQEkWBgYzisQ66sWJkt0AM5JI,1892
|
|
12
|
-
proxy_reader-0.1.0.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
|
|
13
|
-
proxy_reader-0.1.0.dist-info/top_level.txt,sha256=NIrKvLf5DF2_oEnj0yg84roFna2-MaJ7zE1jY2CIU0I,13
|
|
14
|
-
proxy_reader-0.1.0.dist-info/RECORD,,
|
|
File without changes
|