proxy-reader 0.1.0__py3-none-any.whl

proxy_reader/logger.py

@@ -0,0 +1,21 @@
+
+
+import logging
+from logging.handlers import RotatingFileHandler
+
+filename = "proxy_reader.log"
+logging_format = logging.Formatter(
+    "%(levelname)s:[%(filename)s:%(lineno)s]:%(asctime)s: %(message)s")
+logger = logging.getLogger(__name__)
+console_handler = logging.StreamHandler()
+console_handler.setFormatter(logging_format)
+file_handler = RotatingFileHandler(filename, mode='a', maxBytes=5 * 1024 * 1024,
+                                   backupCount=2, encoding="utf-8", delay=False)
+file_handler.setFormatter(logging_format)
+
+logger.setLevel(logging.DEBUG)
+file_handler.setLevel(logging.DEBUG)
+console_handler.setLevel(logging.DEBUG)
+
+logger.addHandler(console_handler)
+logger.addHandler(file_handler)

proxy_reader/protocols/reader.py

@@ -0,0 +1,106 @@
+from typing import List, Optional, Dict, Any, Protocol
+from .._types import ProxiesList
+from ..proxy import Proxy
+
+
+class ProxiesReaderProtocol(Protocol):
+    @property
+    def total(self) -> int:
+        pass
+
+    @property
+    def total_working(self) -> int:
+        pass
+
+    @property
+    def total_bad(self) -> int:
+        pass
+
+    @property
+    def proxies(self) -> ProxiesList:
+        pass
+
+    @property
+    def bad_proxies(self) -> ProxiesList:
+        pass
+
+    @property
+    def working_proxies(self) -> ProxiesList:
+        pass
+
+    @working_proxies.setter
+    def working_proxies(self, working_proxies: List[Proxy]) -> None:
+        pass
+
+    def read_raw(self) -> List[str]:
+        pass
+
+    def random_url(self) -> str:
+        pass
+
+    def read_with_auth(self) -> None:
+        pass
+
+    def read_authless(self) -> None:
+        pass
+
+    async def _check_proxy(
+        self, proxy: Proxy, response_time: Optional[int] = None
+    ) -> bool:
+        pass
+
+    async def check_all_proxies(self, max_resp_time: int = 30) -> None:
+        pass
+
+    async def _check_proxy_socks(
+        self, proxy: Proxy, response_time: Optional[int] = None
+    ) -> bool:
+        pass
+
+    async def check_all_proxies_socks5(self, max_resp_time: int = 5) -> None:
+        pass
+
+    def get_working_proxies_list_http(self) -> List[str]:
+        pass
+
+    def write_working_proxies(self, filename: str) -> None:
+        pass
+
+    def get_random_http(self) -> Optional[str]:
+        pass
+
+    def get_random_socks5(self) -> Optional[str]:
+        pass
+
+    def get_random_socks5_telegram(self) -> Optional[Dict[str, Any]]:
+        pass
+
+    def next_http_from_list(self) -> Optional[str]:
+        pass
+
+    def next_http_from_cycle(self) -> str:
+        pass
+
+    def next_socks5_from_list(self) -> str:
+        pass
+
+    def next_socks5_from_cycle(self) -> str:
+        pass
+
+    def next_http_telegram_from_list(self) -> Dict[str, Any]:
+        pass
+
+    def next_http_telegram_from_cycle(self) -> Dict[str, Any]:
+        pass
+
+    def next_socks5_telegram_from_cycle(self) -> Dict[str, Any]:
+        pass
+
+    def next_socks5_telegram_from_list(self) -> Dict[str, Any]:
+        pass
+
+    def next_https_from_list(self) -> str:
+        pass
+
+    def next_https_from_cycle(self) -> str:
+        pass

proxy_reader/proxy.py

@@ -0,0 +1,78 @@
+from typing import Optional, Dict, Any
+
+
+class Proxy:
+    def __init__(self, ip: str, port: str, username: Optional[str] = None, password: Optional[str] = None) -> None:
+        self._ip = ip
+        self._port = port
+        self._username = username
+        self._password = password
+
+    @property
+    def ip(self) -> str:
+        return self._ip
+
+    @property
+    def port(self) -> str:
+        return self._port
+
+    @property
+    def username(self) -> Optional[str]:
+        return self._username
+
+    @property
+    def password(self) -> Optional[str]:
+        return self._password
+
+    @property
+    def http(self) -> str:
+        if self.username and self.password:
+            return f"http://{self._username}:{self._password}@{self._ip}:{self._port}"
+        return f"http://{self._ip}:{self._port}"
+
+    @property
+    def https(self) -> str:
+        if self.username and self.password:
+            return f"https://{self._username}:{self._password}@{self._ip}:{self._port}"
+        return f"https://{self._ip}:{self._port}"
+
+    @property
+    def telegram_http(self) -> Dict[str, Any]:
+        p = {"proxy_type": 3, "addr": self._ip, "port": int(self._port)}
+        if self.username and self.password:
+            p.update({"username": self.username, "password": self.password})
+        return p
+
+    @property
+    def telegram_socks5(self) -> Dict[str, Any]:
+        p = {"proxy_type": 2, "addr": self._ip, "port": int(self._port)}
+        if self.username and self.password:
+            p.update({"username": self.username, "password": self.password})
+        return p
+
+    @property
+    def telegram_socks4(self) -> Dict[str, Any]:
+        p = {"proxy_type": 1, "addr": self._ip, "port": int(self._port)}
+        if self.username and self.password:
+            p.update({"username": self.username, "password": self.password})
+        return p
+
+    @property
+    def socks5(self) -> str:
+        if self._username is not None and self._password is not None:
+            return f"socks5://{self._username}:{self._password}@{self.ip}:{self.port}"
+        return f"socks5://{self.ip}:{self.port}"
+
+    @property
+    def socks4(self) -> str:
+        if self._username is not None and self._password is not None:
+            return f"socks4://{self._username}:{self._password}@{self.ip}:{self.port}"
+        return f"socks4://{self.ip}:{self.port}"
+
+    def __str__(self) -> str:
+        if self._username is not None and self._password is not None:
+            return f"{self._ip}:{self._port}:{self._username}:{self._password}"
+        return f"{self._ip}:{self._port}"
+
+    def __repr__(self) -> str:
+        return self.__str__()

proxy_reader/reader.py

@@ -0,0 +1,361 @@
+import random
+import itertools
+from .proxy import Proxy
+import aiohttp
+import asyncio
+from .logger import logger, console_handler, file_handler
+import os
+from aiohttp_socks import ProxyConnector
+import sys
+from typing import Optional, List, Dict, Any
+from ._types import ProxiesList, ProxyiesGen
+from .protocols.reader import ProxiesReaderProtocol
+
+
+class ProxiesReader(ProxiesReaderProtocol):
+    def __init__(
+        self,
+        file_path: str = "proxies.txt",
+        shuffle: bool = False,
+        debug: bool = False,
+        extra_debug: bool = False,
+    ) -> None:
+        self._file_path = file_path
+        self._debug = debug
+        self._extra_debug = extra_debug
+
+        self._shuffle = shuffle
+        self._proxies: ProxiesList = []
+        self._has_auth = False
+        self._bad_proxies: ProxiesList = []
+        self._working_proxies: ProxiesList = []
+        self._proxies_checked = False
+        self._proxy_iterator: Optional[ProxyiesGen] = None
+        self._proxy_iterator_cycle: Optional[ProxyiesGen] = None
+        self._thread_control: asyncio.Semaphore = asyncio.Semaphore(500)
+        self._max_response_time = 60
+        self._timeout_count = 0
+
+        self._check_urls = [
+            # New
+            "http://checkip.amazonaws.com",
+            "http://ipinfo.io/ip",
+            "http://icanhazip.com",
+            "http://api.ipify.org",
+            "http://ifconfig.co/ip",
+            # "http://ipapi.co/ip",
+            "http://ident.me",
+            "http://api64.ipify.org?format=json",
+            "http://ifconfig.me/ip",
+            "http://www.trackip.net/ip",
+            # "http://api.adviceslip.com/advice",
+            # "http://api.thecatapi.com/v1/images/search",
+            # "http://dog.ceo/api/breeds/image/random",
+        ]
+
+        if not self._debug:
+            logger.removeHandler(file_handler)
+            logger.removeHandler(console_handler)
+            try:
+                os.remove(file_handler.baseFilename)
+            except Exception:
+                pass
+
+    @property
+    def total(self) -> int:
+        return len(self._proxies)
+
+    @property
+    def total_working(self) -> int:
+        return len(self._working_proxies)
+
+    @property
+    def total_bad(self) -> int:
+        return len(self._bad_proxies)
+
+    @property
+    def proxies(self) -> ProxiesList:
+        return self._proxies
+
+    @property
+    def bad_proxies(self) -> ProxiesList:
+        return self._bad_proxies
+
+    @property
+    def working_proxies(self) -> ProxiesList:
+        return self._working_proxies
+
+    @working_proxies.setter
+    def working_proxies(self, working_proxies: list[Proxy]) -> None:
+        self._working_proxies = working_proxies
+
+    def __str__(self) -> str:
+        return str(self._proxies)
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+    def read_raw(self) -> List[str]:
+        lines = open(self._file_path).readlines()
+        return [line.strip().replace("\n", "") for line in lines]
+
+    def random_url(self) -> str:
+        return random.choice(self._check_urls)
+
+    def read_with_auth(self) -> None:
+        """Format: IP:PORT:USERNAME:PASSWORD"""
+        raw_proxies = self.read_raw()
+        for proxy in raw_proxies:
+            sp_proxy = proxy.split(":")
+            ip = sp_proxy[0]
+            port = sp_proxy[1]
+            username = sp_proxy[2]
+            password = sp_proxy[3]
+            self._proxies.append(Proxy(ip, port, username, password))
+
+        self._has_auth = True
+        if self._shuffle:
+            random.shuffle(self._proxies)
+
+    def read_authless(self) -> None:
+        """Format: IP:PORT"""
+        raw_proxies = self.read_raw()
+        for proxy in raw_proxies:
+            sp_proxy = proxy.split(":")
+            ip = sp_proxy[0]
+            port = sp_proxy[1]
+            self._proxies.append(Proxy(ip, port))
+        logger.debug(
+            f"Loaded total {len(self._proxies)} proxies from {self._file_path}"
+        )
+        if self._shuffle:
+            random.shuffle(self._proxies)
+
+    async def _check_proxy(
+        self, proxy: Proxy, response_time: Optional[int] = None
+    ) -> bool:
+        connectins_limit = 60 if "win" in sys.platform else 100
+        connector = aiohttp.TCPConnector(limit=connectins_limit)
+        session = aiohttp.ClientSession(connector=connector)
+        url = self.random_url()
+        p = proxy.http
+
+        async with self._thread_control:
+            logger.debug(f"Checking proxy {p} ..")
+            try:
+                # resp = await asyncio.wait_for(session.get(url, proxy=p), timeout=self._max_response_time)
+                resp = await session.get(
+                    url,
+                    timeout=aiohttp.ClientTimeout(self._max_response_time),
+                    proxy=p,
+                    ssl=False,
+                )
+                # await resp.read()
+
+            except asyncio.TimeoutError as e:
+                self._timeout_count += 1
+                logger.debug(f"{p} : TIMEOUT {e}. {url}")
+                self._bad_proxies.append(proxy)
+                await session.close()
+                return False
+
+            except Exception as e:
+                logger.debug(f"Bad proxy raised. {e}", exc_info=self._extra_debug)
+                await session.close()
+                return False
+
+            if resp.status == 200:
+                logger.debug(f"{p}: Working")
+                self._working_proxies.append(proxy)
+
+            else:
+                logger.debug(f"{p}: Not Working. Response code: {resp.status}")
+                self._bad_proxies.append(proxy)
+
+            return True
+
+    async def check_all_proxies(self, max_resp_time: int = 30) -> None:
+        """Run this to check all proxies at once."""
+        tasks: List[asyncio.Task[bool]] = []
+        for proxy in self._proxies:
+            tasks.append(asyncio.create_task(self._check_proxy(proxy, max_resp_time)))
+        await asyncio.gather(*tasks)
+        self._proxies_checked = True
+        logger.debug("All proxies checked.")
+
+    async def _check_proxy_socks(
+        self, proxy: Proxy, response_time: Optional[int] = None
+    ) -> bool:
+        url = self.random_url()
+        socks_connector = ProxyConnector.from_url(proxy.socks5)  # type: ignore
+        session = aiohttp.ClientSession(connector=socks_connector)
+        logger.debug(f"Checking proxy {proxy} ..")
+        try:
+            resp = await asyncio.wait_for(session.get(url), timeout=response_time)
+
+        except asyncio.TimeoutError:
+            logger.debug(f"{proxy} : TIMEOUT: Not working.")
+            self._bad_proxies.append(proxy)
+            await session.close()
+            return False
+
+        except Exception as e:
+            logger.debug(f"Bad proxy raised. {e}", exc_info=self._extra_debug)
+            await session.close()
+            return False
+
+        await resp.read()
+        await session.close()
+        if resp.status == 200:
+            logger.debug(f"{proxy}: Working")
+            self._working_proxies.append(proxy)
+        else:
+            logger.debug(f"{proxy}: Not Working")
+            self._bad_proxies.append(proxy)
+
+        return True
+
+    async def check_all_proxies_socks5(self, max_resp_time: int = 5) -> None:
+        """Run the check on all proxies at once."""
+        tasks: List[asyncio.Task[bool]] = []
+        for proxy in self._proxies:
+            tasks.append(
+                asyncio.create_task(self._check_proxy_socks(proxy, max_resp_time))
+            )
+        await asyncio.gather(*tasks)
+        self._proxies_checked = True
+        logger.debug("All proxies checked.")
+
+    def get_working_proxies_list_http(self) -> List[str]:
+        working_list: List[str] = []
+        for proxy in self._working_proxies:
+            working_list.append(proxy.http)
+        return working_list
+
+    def write_working_proxies(self, filename: str) -> None:
+        working_list = self.get_working_proxies_list_http()
+        logger.debug(working_list)
+        with open(filename, "w") as f:
+            f.write("\n".join([proxy.strip() for proxy in working_list]))
+        logger.debug(f"Proxies written to: {filename}")
+
+    def get_random_http(self) -> Optional[str]:
+        _p = None
+        if len(self._working_proxies) > 0:
+            proxy = random.choice(self._working_proxies)
+            _p = proxy.http
+        return _p
+
+    def get_random_socks5(self) -> Optional[str]:
+        _p = None
+        if len(self._working_proxies) > 0:
+            proxy = random.choice(self._working_proxies)
+            _p = proxy.socks5
+        return _p
+
+    def get_random_socks5_telegram(self) -> Optional[Dict[str, Any]]:
+        _p = None
+        if len(self._working_proxies) > 0:
+            proxy = random.choice(self._working_proxies)
+            _p = proxy.telegram_socks5
+        return _p
+
+    def next_http_from_list(self) -> Optional[str]:
+        """Get next proxy from proxies list"""
+
+        def __iter() -> ProxyiesGen:
+            for proxy in self._working_proxies:
+                yield proxy
+
+        if self._proxy_iterator is None:
+            self._proxy_iterator = __iter()
+
+        return str(next(self._proxy_iterator).http)
+
+    def next_http_from_cycle(self) -> str:
+        """Get next proxy from proxies cycle"""
+
+        def __iter() -> ProxyiesGen:
+            for proxy in itertools.cycle(self._working_proxies):
+                yield proxy
+
+        if self._proxy_iterator_cycle is None:
+            self._proxy_iterator_cycle = __iter()
+
+        return str(next(self._proxy_iterator_cycle).http)
+
+    def next_socks5_from_list(self) -> str:
+        """Get next proxy from proxies list"""
+
+        if self._proxy_iterator is None:
+            self._proxy_iterator = self.__proxies_gen(self._working_proxies)
+        return str(next(self._proxy_iterator).socks5)
+
+    def next_socks5_from_cycle(self) -> str:
+        """Get next proxy from proxies cycle"""
+
+        if self._proxy_iterator_cycle is None:
+            self._proxy_iterator_cycle = self.__proxies_gen(
+                self._working_proxies, cycle_proxies=True
+            )
+        return str(next(self._proxy_iterator_cycle).socks5)
+
+    def next_http_telegram_from_list(self) -> Dict[str, Any]:
+        """Get next proxy from proxies list"""
+
+        if self._proxy_iterator is None:
+            self._proxy_iterator = self.__proxies_gen(self._working_proxies)
+        return dict(next(self._proxy_iterator).telegram_http)
+
+    def next_http_telegram_from_cycle(self) -> Dict[str, Any]:
+        """Get next proxy from proxies cycle"""
+
+        if self._proxy_iterator is None:
+            self._proxy_iterator = self.__proxies_gen(
+                self._working_proxies, cycle_proxies=True
+            )
+        return dict(next(self._proxy_iterator).telegram_http)
+
+    def next_socks5_telegram_from_cycle(self) -> Dict[str, Any]:
+        """Get next proxy from proxies cycle"""
+        if self._proxy_iterator_cycle is None:
+            self._proxy_iterator_cycle = self.__proxies_gen(
+                self._working_proxies, cycle_proxies=True
+            )
+        return dict(next(self._proxy_iterator_cycle).telegram_socks5)
+
+    def next_socks5_telegram_from_list(self) -> Dict[str, Any]:
+        """Get next proxy from proxies cycle"""
+
+        if self._proxy_iterator_cycle is None:
+            self._proxy_iterator_cycle = self.__proxies_gen(self._working_proxies)
+        return dict(next(self._proxy_iterator_cycle).telegram_socks5)
+
+    def next_https_from_list(self) -> str:
+        """Get next proxy from proxies list"""
+
+        if self._proxy_iterator is None:
+            self._proxy_iterator = self.__proxies_gen(self._working_proxies)
+
+        return str(next(self._proxy_iterator).https)
+
+    def next_https_from_cycle(self) -> str:
+        """Get next proxy from proxies cycle"""
+
+        if not self._proxy_iterator_cycle:
+            self._proxy_iterator_cycle = self.__proxies_gen(
+                self._working_proxies, cycle_proxies=True
+            )
+
+        return str(next(self._proxy_iterator_cycle).https)
+
+    def __proxies_gen(
+        self, proxies_list: List[Proxy], cycle_proxies: bool = False
+    ) -> ProxyiesGen:
+        if cycle_proxies:
+            for proxy in itertools.cycle(self._working_proxies):
+                yield proxy
+
+        else:
+            for proxy in proxies_list:
+                yield proxy

proxy_reader-0.1.0.dist-info/LICENSE

@@ -0,0 +1,8 @@
+The MIT License (MIT)
+Copyright © 2024 Rune Tech
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

proxy_reader-0.1.0.dist-info/METADATA

@@ -0,0 +1,33 @@
+Metadata-Version: 2.1
+Name: proxy-reader
+Version: 0.1.0
+Summary: Read and check bulk proxies effectively
+Author-email: Rune Tech <runetech2024@gmail.com>
+License: The MIT License (MIT)
+        Copyright © 2024 Rune Tech
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+Project-URL: Homepage, https://github.com/runetech0/proxy-reader
+Project-URL: github, https://github.com/runetech0/proxy-reader
+Keywords: proxies,proxy,reader,checker,bulk
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiohttp >=3.10.10
+Requires-Dist: aiohttp-socks >=0.9.0
+
+
+
+
+# proxy_reader - A simple but useful bulk proxies reader and checker.
+
+This is useful when you are working with multiple proxies and want to bulk check
+these proxies before using them.
+It has iterators to iterate the proxies for easy reading.
+
+
+## Changelogs

proxy_reader-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+proxy_reader/__init__.py,sha256=8_HgszorTyU_0OJmvxgimP9S21y07s9R1n_U-wHMwTs,46
+proxy_reader/_types.py,sha256=OWAYiN91E_IWgs1myPy1Q9l-8XkVVq8uDzjLOTB_Eww,317
+proxy_reader/domains.py,sha256=d4FUb2zw3Kg9bAjtaggEcxZA293WvVcRTgMKKPmYr1A,135662
+proxy_reader/logger.py,sha256=BxaHptwZlPPGdx5mbS8vi77u0N-W6cKhcR-5lG2x93c,710
+proxy_reader/proxy.py,sha256=vkI2vonxlm0x0u3Tojq6xKaw5_7fvSbmICW8b_M4lzY,2645
+proxy_reader/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+proxy_reader/reader.py,sha256=KeaAPhxMI6moYaLmzYj7ZGcn8REXWs-1xBMkXuK2Ff0,12373
+proxy_reader/protocols/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+proxy_reader/protocols/reader.py,sha256=WscEXPpKmJuMF2R7EBqDZTheRh8BIUilukojcPRFyRg,2303
+proxy_reader-0.1.0.dist-info/LICENSE,sha256=8DyHi9RsZXeM8s1DpaXubZSSG44PkzV2oskJE3E09F4,1081
+proxy_reader-0.1.0.dist-info/METADATA,sha256=hsWGBfcZrYHMTZWKY_oQEkWBgYzisQ66sWJkt0AM5JI,1892
+proxy_reader-0.1.0.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+proxy_reader-0.1.0.dist-info/top_level.txt,sha256=NIrKvLf5DF2_oEnj0yg84roFna2-MaJ7zE1jY2CIU0I,13
+proxy_reader-0.1.0.dist-info/RECORD,,

proxy_reader-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.2.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

proxy_reader-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+proxy_reader