swiftshadow 2.1.0__py3-none-any.whl → 2.2.0__py3-none-any.whl

swiftshadow/classes.py

@@ -1,19 +1,20 @@
+from asyncio import run
 from datetime import datetime
+from logging import DEBUG, INFO, FileHandler, Formatter, StreamHandler, getLogger
+from pathlib import Path
+from pickle import dump, dumps, load, loads
 from random import choice
+from sys import stdout
 from typing import Literal
-from pathlib import Path
+
+import aiofiles
 from appdirs import user_cache_dir
-from logging import FileHandler, getLogger, Formatter, StreamHandler, INFO, DEBUG
-from sys import stdout
-from pickle import load, dump, loads, dumps
-from swiftshadow.cache import checkExpiry, getExpiry
-from swiftshadow.models import CacheData, Proxy as Proxy
 
+from swiftshadow.cache import checkExpiry, getExpiry
 from swiftshadow.exceptions import UnsupportedProxyProtocol
+from swiftshadow.models import CacheData
+from swiftshadow.models import Proxy as Proxy
 from swiftshadow.providers import Providers
-from asyncio import run
-import aiofiles
-
 
 logger = getLogger("swiftshadow")
 logger.setLevel(INFO)
@@ -87,13 +88,16 @@ class ProxyInterface:
 
         if protocol not in ["https", "http"]:
             raise UnsupportedProxyProtocol(
-                f"Protocol {protocol} is not supported by swiftshadow, please choose between HTTP or HTTPS"
+                f"Protocol {
+                    protocol
+                } is not supported by swiftshadow, please choose between HTTP or HTTPS"
             )
         self.protocol: Literal["https", "http"] = protocol
 
         self.maxproxies: int = maxProxies
         self.autorotate: bool = autoRotate
         self.cachePeriod: int = cachePeriod
+        self.configString: str = f"{maxProxies}{''.join(protocol)}{''.join(countries)}"
 
         if debug:
             logger.setLevel(DEBUG)
@@ -136,11 +140,15 @@ class ProxyInterface:
                 pickled_bytes = await cacheFile.read()
                 cache: CacheData = loads(pickled_bytes)
 
-                if not checkExpiry(cache.expiryIn):
+                if self.configString != cache.configString:
+                    logger.info("Cache Invalid due to configuration changes.")
+                elif not checkExpiry(cache.expiryIn):
                     self.proxies = cache.proxies
                     logger.info("Loaded proxies from cache.")
                     logger.debug(
-                        f"Cache with {len(cache.proxies)} proxies, expire in {cache.expiryIn}"
+                        f"Cache with {len(cache.proxies)} proxies, expire in {
+                            cache.expiryIn
+                        }"
                     )
                     self.current = self.proxies[0]
                     self.cacheExpiry = cache.expiryIn
@@ -162,7 +170,9 @@ class ProxyInterface:
                 self.countries, self.protocol
             )
             logger.debug(
-                f"{len(providerProxies)} proxies from {provider.providerFunction.__name__}"
+                f"{len(providerProxies)} proxies from {
+                    provider.providerFunction.__name__
+                }"
             )
             self.proxies.extend(providerProxies)
 
@@ -170,14 +180,16 @@ class ProxyInterface:
                 break
 
         if len(self.proxies) == 0:
-            raise ValueError("No proxies where found for the current filter settings.")
+            if self.protocol == "https":
+                raise ValueError("No proxies were found for the current filter settings. Tip: https proxies can be rare; recommend setting protocol to http")
+            raise ValueError("No proxies were found for the current filter settings.")
 
         async with aiofiles.open(
             self.cacheFolderPath.joinpath("swiftshadow.pickle"), "wb+"
         ) as cacheFile:
             cacheExpiry = getExpiry(self.cachePeriod)
             self.cacheExpiry = cacheExpiry
-            cache = CacheData(cacheExpiry, self.proxies)
+            cache = CacheData(cacheExpiry, self.configString, self.proxies)
             pickled_bytes = dumps(cache)
             _ = await cacheFile.write(pickled_bytes)
         self.current = self.proxies[0]
@@ -199,11 +211,15 @@ class ProxyInterface:
             ) as cacheFile:
                 cache: CacheData = load(cacheFile)
 
-                if not checkExpiry(cache.expiryIn):
+                if self.configString != cache.configString:
+                    logger.info("Cache Invalid due to configuration changes.")
+                elif not checkExpiry(cache.expiryIn):
                     self.proxies = cache.proxies
                     logger.info("Loaded proxies from cache.")
                     logger.debug(
-                        f"Cache with {len(cache.proxies)} proxies, expire in {cache.expiryIn}"
+                        f"Cache with {len(cache.proxies)} proxies, expire in {
+                            cache.expiryIn
+                        }"
                     )
                     self.current = self.proxies[0]
                     logger.debug(f"Cache set to expire at {cache.expiryIn}")
@@ -225,7 +241,9 @@ class ProxyInterface:
                 provider.providerFunction(self.countries, self.protocol)
             )
             logger.debug(
-                f"{len(providerProxies)} proxies from {provider.providerFunction.__name__}"
+                f"{len(providerProxies)} proxies from {
+                    provider.providerFunction.__name__
+                }"
             )
             self.proxies.extend(providerProxies)
 
@@ -233,14 +251,14 @@ class ProxyInterface:
                 break
 
         if len(self.proxies) == 0:
-            raise ValueError("No proxies where found for the current filter settings.")
+            raise ValueError("No proxies were found for the current filter settings.")
 
         with open(
             self.cacheFolderPath.joinpath("swiftshadow.pickle"), "wb+"
         ) as cacheFile:
             cacheExpiry = getExpiry(self.cachePeriod)
             self.cacheExpiry = cacheExpiry
-            cache = CacheData(cacheExpiry, self.proxies)
+            cache = CacheData(cacheExpiry, self.configString, self.proxies)
             dump(cache, cacheFile)
         self.current = self.proxies[0]
 

swiftshadow/helpers.py

@@ -4,6 +4,7 @@ from typing import Literal
 from requests import get
 
 from swiftshadow.models import Proxy
+from swiftshadow.validator import validate_proxies
 
 
 def checkProxy(proxy):
@@ -36,3 +37,12 @@ def plaintextToProxies(text: str, protocol: Literal["http", "https"]) -> list[Pr
         proxy = Proxy(ip=ip, port=int(port), protocol=protocol)
         proxies.append(proxy)
     return proxies
+
+
+async def GenericPlainTextProxyProvider(
+    url: str, protocol: Literal["http", "https"] = "http"
+) -> list[Proxy]:
+    raw: str = get(url).text
+    proxies: list[Proxy] = plaintextToProxies(raw, protocol=protocol)
+    results = await validate_proxies(proxies)
+    return results

swiftshadow/models.py

@@ -46,10 +46,12 @@ class CacheData:
 
     Attributes:
         expiryIn: Expiry date object.
+        configString: Configuration String for the ProxyInterface this cache was created.
         proxies: Proxies to head.
     """
 
     expiryIn: datetime
+    configString: str
     proxies: list[Proxy]
 
 

swiftshadow/providers.py

@@ -1,11 +1,14 @@
 from typing import Literal
+import aiohttp
 
 from requests import get
 
-from swiftshadow.helpers import plaintextToProxies
+from swiftshadow.helpers import GenericPlainTextProxyProvider
 from swiftshadow.models import Provider, Proxy
+from asyncio import create_task, gather
 from swiftshadow.types import MonosansProxyDict
 from swiftshadow.validator import validate_proxies
+from lxml import etree
 
 
 async def Monosans(
@@ -38,11 +41,10 @@ async def Monosans(
 async def Thespeedx(
     countries: list[str] = [], protocol: Literal["http", "https"] = "http"
 ):
-    raw: str = get(
-        "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt"
-    ).text
-    proxies: list[Proxy] = plaintextToProxies(raw, protocol="http")
-    results = await validate_proxies(proxies)
+    results = await GenericPlainTextProxyProvider(
+        url="https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt",
+        protocol="http",
+    )
     return results
 
 
@@ -83,63 +85,91 @@ async def GoodProxy(
 async def OpenProxyList(
     countries: list[str] = [], protocol: Literal["http", "https"] = "http"
 ):
-    raw = get("https://api.openproxylist.xyz/http.txt").text
-    proxies: list[Proxy] = plaintextToProxies(raw, protocol="http")
-    results = await validate_proxies(proxies)
+    results = await GenericPlainTextProxyProvider(
+        "https://api.openproxylist.xyz/http.txt", "http"
+    )
     return results
 
 
 async def MuRongPIG(
     countries: list[str] = [], protocol: Literal["http", "https"] = "http"
 ):
-    raw = get(
-        "https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/refs/heads/main/http_checked.txt"
-    ).text
-    proxies: list[Proxy] = plaintextToProxies(raw, protocol="http")
-    results = await validate_proxies(proxies)
+    results = await GenericPlainTextProxyProvider(
+        "https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/refs/heads/main/http_checked.txt",
+        "http",
+    )
     return results
 
 
-async def KangProxy(
+async def Mmpx12(
     countries: list[str] = [], protocol: Literal["http", "https"] = "http"
 ):
-    raw = get(
-        f"https://github.com/officialputuid/KangProxy/raw/refs/heads/KangProxy/{protocol}/{protocol}.txt"
-    ).text
-    proxies: list[Proxy] = plaintextToProxies(raw, protocol=protocol)
-    results = await validate_proxies(proxies)
+    url = f"https://github.com/mmpx12/proxy-list/raw/refs/heads/master/{protocol}.txt"
+    results = await GenericPlainTextProxyProvider(url, protocol)
     return results
 
 
-async def Mmpx12(
+async def Anonym0usWork1221(
     countries: list[str] = [], protocol: Literal["http", "https"] = "http"
 ):
-    raw = get(
-        f"https://github.com/mmpx12/proxy-list/raw/refs/heads/master/{protocol}.txt"
-    ).text
-    proxies: list[Proxy] = plaintextToProxies(raw, protocol=protocol)
-    results = await validate_proxies(proxies)
+    url = f"https://github.com/Anonym0usWork1221/Free-Proxies/raw/refs/heads/main/proxy_files/{protocol}_proxies.txt"
+    results = await GenericPlainTextProxyProvider(url, protocol)
     return results
 
 
-async def Anonym0usWork1221(
+async def ProxySpace(
     countries: list[str] = [], protocol: Literal["http", "https"] = "http"
 ):
-    raw = get(
-        f"https://github.com/Anonym0usWork1221/Free-Proxies/raw/refs/heads/main/proxy_files/{protocol}_proxies.txt"
-    ).text
-    proxies: list[Proxy] = plaintextToProxies(raw, protocol=protocol)
-    results = await validate_proxies(proxies)
+    results = await GenericPlainTextProxyProvider(
+        "https://proxyspace.pro/http.txt", "http"
+    )
     return results
 
 
-async def ProxySpace(
+async def ProxyDB(
     countries: list[str] = [], protocol: Literal["http", "https"] = "http"
 ):
-    raw = get("https://proxyspace.pro/http.txt").text
-    proxies: list[Proxy] = plaintextToProxies(raw, protocol="http")
-    results = await validate_proxies(proxies)
-    return results
+    base_url = f"https://www.proxydb.net/?protocol={protocol}&sort_column_id=uptime&sort_order_desc=true"
+    proxies: list[Proxy] = []
+    raw = get(base_url).text
+    total = int(
+        raw.split("Showing")[-1].split("total proxies")[0].split("of")[-1].strip()
+    )
+
+    async def parsePage(session: aiohttp.ClientSession, url: str):
+        proxies = []
+        async with session.get(url) as resp:
+            raw = await resp.text()
+            exml = etree.HTML(raw)
+            table = exml.find("body/div/div/table/tbody")
+            rows = iter(table)
+            for row in rows:
+                if len(proxies) > 500:
+                    break
+                data = []
+                for td in row[:4]:
+                    text = td.text.strip()
+                    if text == "":
+                        text = list(td)[-1].text
+                        data.append(text)
+                if countries != [] and data[-1] not in countries:
+                    continue
+                proxy = Proxy(data[0], protocol, data[1])
+                proxies.append(proxy)
+        return proxies
+
+    tasks = []
+    async with aiohttp.ClientSession() as session:
+        for offset in range(0, total, 30):
+            url = base_url + f"&offset={offset}"
+            task = create_task(coro=parsePage(session, url))
+            tasks.append(task)
+        results = await gather(*tasks, return_exceptions=True)
+    for result in results:
+        if isinstance(result, BaseException):
+            continue
+        proxies.extend(result)
+    return proxies
 
 
 Providers: list[Provider] = [
@@ -154,9 +184,7 @@ Providers: list[Provider] = [
     ),
     Provider(providerFunction=Mmpx12, countryFilter=False, protocols=["http", "https"]),
     Provider(providerFunction=GoodProxy, countryFilter=False, protocols=["http"]),
-    Provider(
-        providerFunction=KangProxy, countryFilter=False, protocols=["http", "https"]
-    ),
     Provider(providerFunction=ProxySpace, countryFilter=False, protocols=["http"]),
     Provider(providerFunction=OpenProxyList, countryFilter=False, protocols=["http"]),
+    Provider(providerFunction=ProxyDB, countryFilter=True, protocols=["http", "https"]),
 ]

swiftshadow/validator.py

@@ -42,21 +42,25 @@ async def get_host_ip(async_session: aiohttp.ClientSession) -> str | None:
         return ip
 
 
-async def check_proxy(async_session: aiohttp.ClientSession, proxy: Proxy) -> str:
+async def check_proxy(
+    async_session: aiohttp.ClientSession, proxy: Proxy, checker: str
+) -> str:
     """
-    Check one proxy abject.
+    Check one proxy object.
 
     Args:
         async_session: aiohttp client session object
         proxy: Proxy Object
+        checker: Proxy checker API.
 
     Returns:
         text: API response text
     """
     async with async_session.get(
-        url=f"{proxy.protocol}://checkip.amazonaws.com",
+        url=f"{proxy.protocol}://{checker}",
         proxy=proxy.as_string(),
-        timeout=4,
+        timeout=5,
+        ssl=False,
     ) as response:
         text = await response.text()
         return text
@@ -73,16 +77,33 @@ async def validate_proxies(proxies: list[Proxy]) -> list[Proxy]:
         working_proxies: List of working Proxies
     """
     working_proxies: list[Proxy] = []
-    async with aiohttp.ClientSession() as async_session:
+    checkers = [
+        "checkip.amazonaws.com",
+        "ipinfo.io/ip",
+        "api.ipify.org/",
+        "whatsmyip.dev/api/ip",
+        "ip4.anysrc.net/banner",
+        "api4.my-ip.io/v2/ip.txt",
+        "api.myip.la",
+        "api.seeip.org",
+        "ips.im/api",
+        "ifconfig.me/ip",
+        "myip.expert/api/",
+        "checkip.info/ip",
+        "api.myip.com",
+    ]
+    total_checkers = len(checkers)
+    tcp_connection = aiohttp.TCPConnector(limit=100)
+    async with aiohttp.ClientSession(connector=tcp_connection) as async_session:
         tasks = []
 
         host_task = asyncio.create_task(coro=get_host_ip(async_session))
         tasks.append(host_task)
 
-        for proxy in proxies:
-            task = asyncio.create_task(coro=check_proxy(async_session, proxy))
+        for idx, proxy in enumerate(proxies):
+            checker = checkers[idx % total_checkers]
+            task = asyncio.create_task(coro=check_proxy(async_session, proxy, checker))
             tasks.append(task)
-
         results = await asyncio.gather(*tasks, return_exceptions=True)
         host_ip = results[0]
         results = results[1:]

{swiftshadow-2.1.0.dist-info → swiftshadow-2.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: swiftshadow
-Version: 2.1.0
+Version: 2.2.0
 Summary: Free IP Proxy rotator for python
 Author-email: sachin-sankar <mail.sachinsankar@gmail.com>
 License-File: LICENSE
@@ -8,6 +8,7 @@ Requires-Python: >=3.12
 Requires-Dist: aiofiles>=24.1.0
 Requires-Dist: aiohttp>=3.11.11
 Requires-Dist: appdirs>=1.4.4
+Requires-Dist: lxml>=6.0.0
 Requires-Dist: requests>=2.32.3
 Description-Content-Type: text/markdown
 

swiftshadow-2.2.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+swiftshadow/__init__.py,sha256=oxJDmG0YO0XyYGQpa9V_o38lZnoie7U_d4Ik_h46UXs,922
+swiftshadow/cache.py,sha256=Mg8xsD6K3K012sILBwD2EZH6CE5kWCQNKCfZ5yadalI,800
+swiftshadow/classes.py,sha256=hXAyvRuGXDvj_TsZxJ-bapK0pL82lhZUBGQZbQiGWN0,12280
+swiftshadow/exceptions.py,sha256=qu4eXyrkWD9qd4HCIR-8vRfVcqLlTupo4sD72alCdug,129
+swiftshadow/helpers.py,sha256=b8my66sBw_lTVLaKu4SaGjYQqEfIaStsuZlmK9Pmy5Y,1294
+swiftshadow/models.py,sha256=rHOuOFc6UYCI8L2pwxAbvS3Fj0Ag89cHod0rt7kQ2Vc,1790
+swiftshadow/providers.py,sha256=3IjITgKyywynssYKmIUP5jbcmbezIlaJVR7ODcBiDiE,6628
+swiftshadow/types.py,sha256=Alyw3n54OESX1vSR-0kTvpYTlJ8LKfy5J9WZbtglHpE,894
+swiftshadow/validator.py,sha256=3qE-99uljtbin1125uOgqAjnLRjuIgNYO5iiUAmaBC8,3102
+swiftshadow-2.2.0.dist-info/METADATA,sha256=a5SOKU3R1eBHRA6SovdAYDD5m4qQ3VH3qxmtx8FVSi8,3320
+swiftshadow-2.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+swiftshadow-2.2.0.dist-info/entry_points.txt,sha256=yMj0uEagcmXK2dmMmNXWebTpTT9j5K03oaRrd2wkyLA,49
+swiftshadow-2.2.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+swiftshadow-2.2.0.dist-info/RECORD,,

swiftshadow-2.1.0.dist-info/RECORD

@@ -1,14 +0,0 @@
-swiftshadow/__init__.py,sha256=oxJDmG0YO0XyYGQpa9V_o38lZnoie7U_d4Ik_h46UXs,922
-swiftshadow/cache.py,sha256=Mg8xsD6K3K012sILBwD2EZH6CE5kWCQNKCfZ5yadalI,800
-swiftshadow/classes.py,sha256=nTR_zykns2oxJ4Bj2UXxoXmLqNOMtkLqy17p_LH8odc,11423
-swiftshadow/exceptions.py,sha256=qu4eXyrkWD9qd4HCIR-8vRfVcqLlTupo4sD72alCdug,129
-swiftshadow/helpers.py,sha256=kC5PvfvDCQwigAmfkxlhb4PhcmgdCNpJksMiqyx9lU4,960
-swiftshadow/models.py,sha256=YyfZV98tPdLnF1O3WmTNUNoK4t0GuchfEftzjiM03ck,1678
-swiftshadow/providers.py,sha256=Z_NEOlMx0rr1Z5rfz-rA1kQ4YHKGRuI65jBxMGOyJkE,5660
-swiftshadow/types.py,sha256=Alyw3n54OESX1vSR-0kTvpYTlJ8LKfy5J9WZbtglHpE,894
-swiftshadow/validator.py,sha256=z0dmRKxhvPETSXfC2hTImL0t8pw0zjSYIhUaDMJcJhU,2469
-swiftshadow-2.1.0.dist-info/METADATA,sha256=NTPUMBc5xlCEkcryOEC81Au_MCD8JPO8QxrhkEDo7AM,3293
-swiftshadow-2.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-swiftshadow-2.1.0.dist-info/entry_points.txt,sha256=yMj0uEagcmXK2dmMmNXWebTpTT9j5K03oaRrd2wkyLA,49
-swiftshadow-2.1.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-swiftshadow-2.1.0.dist-info/RECORD,,