swiftshadow 2.0.0__py3-none-any.whl → 2.1.0__py3-none-any.whl

swiftshadow/__init__.py

@@ -1,7 +1,8 @@
+from asyncio import run
 from typing import Literal
-from swiftshadow.providers import Providers
+
 from swiftshadow.models import Proxy
-from asyncio import run
+from swiftshadow.providers import Providers
 
 
 def QuickProxy(

swiftshadow/cache.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timedelta, timezone
 
 
 def getExpiry(timeToLive: int) -> datetime:

swiftshadow/classes.py

@@ -5,13 +5,15 @@ from pathlib import Path
 from appdirs import user_cache_dir
 from logging import FileHandler, getLogger, Formatter, StreamHandler, INFO, DEBUG
 from sys import stdout
-from pickle import load, dump
+from pickle import load, dump, loads, dumps
 from swiftshadow.cache import checkExpiry, getExpiry
 from swiftshadow.models import CacheData, Proxy as Proxy
 
 from swiftshadow.exceptions import UnsupportedProxyProtocol
 from swiftshadow.providers import Providers
 from asyncio import run
+import aiofiles
+
 
 logger = getLogger("swiftshadow")
 logger.setLevel(INFO)
@@ -33,6 +35,7 @@ class ProxyInterface:
         protocol (Literal['https', 'http']): Proxy protocol to use. Defaults to 'http'.
         maxproxies (int): Maximum number of proxies to collect from providers. Defaults to 10.
         autorotate (bool): Whether to automatically rotate proxy on each get() call. Defaults to False.
+        autoUpdate (bool): Whether to automatically update proxies upon class initalisation. Defaults to True.
         cachePeriod (int): Number of minutes before cache is considered expired. Defaults to 10.
         cacheFolderPath (Path): Filesystem path for cache storage. Uses system cache dir by default.
         proxies (list[Proxy]): List of available proxy objects.
@@ -60,6 +63,7 @@ class ProxyInterface:
         protocol: Literal["https", "http"] = "http",
         maxProxies: int = 10,
         autoRotate: bool = False,
+        autoUpdate: bool = True,
         cachePeriod: int = 10,
         cacheFolderPath: Path | None = None,
         debug: bool = False,
@@ -72,6 +76,7 @@ class ProxyInterface:
             protocol: Proxy protocol to retrieve. Choose between 'http' or 'https'.
             maxProxies: Maximum proxies to collect from all providers combined.
             autoRotate: Enable automatic proxy rotation on every get() call.
+            autoUpdate (bool): Whether to automatically update proxies upon class initalisation.
             cachePeriod: Cache validity duration in minutes.
             cacheFolderPath: Custom path for cache storage. Uses system cache dir if None.
             debug: Enable debug logging level when True.
@@ -108,8 +113,74 @@ class ProxyInterface:
         self.proxies: list[Proxy] = []
         self.current: Proxy | None = None
         self.cacheExpiry: datetime | None = None
+        self.autoUpdate = autoUpdate
+
+        if self.autoUpdate:
+            self.update()
+
+    async def async_update(self):
+        """
+        Updates proxy list from providers or cache in async.
+
+        First attempts to load valid proxies from cache. If cache is expired/missing,
+        fetches fresh proxies from registered providers that match country and protocol filters.
+        Updates cache file with new proxies if fetched from providers.
+
+        Raises:
+            ValueError: If no proxies found after provider scraping.
+        """
+        try:
+            async with aiofiles.open(
+                self.cacheFolderPath.joinpath("swiftshadow.pickle"), "rb"
+            ) as cacheFile:
+                pickled_bytes = await cacheFile.read()
+                cache: CacheData = loads(pickled_bytes)
 
-        self.update()
+                if not checkExpiry(cache.expiryIn):
+                    self.proxies = cache.proxies
+                    logger.info("Loaded proxies from cache.")
+                    logger.debug(
+                        f"Cache with {len(cache.proxies)} proxies, expire in {cache.expiryIn}"
+                    )
+                    self.current = self.proxies[0]
+                    self.cacheExpiry = cache.expiryIn
+                    logger.debug(f"Cache set to expire at {cache.expiryIn}")
+                    return
+                else:
+                    logger.info("Cache Expired")
+        except FileNotFoundError:
+            logger.info("No cache found, will be created after update.")
+
+        self.proxies = []
+
+        for provider in Providers:
+            if self.protocol not in provider.protocols:
+                continue
+            if (len(self.countries) != 0) and (not provider.countryFilter):
+                continue
+            providerProxies: list[Proxy] = await provider.providerFunction(
+                self.countries, self.protocol
+            )
+            logger.debug(
+                f"{len(providerProxies)} proxies from {provider.providerFunction.__name__}"
+            )
+            self.proxies.extend(providerProxies)
+
+            if len(self.proxies) >= self.maxproxies:
+                break
+
+        if len(self.proxies) == 0:
+            raise ValueError("No proxies where found for the current filter settings.")
+
+        async with aiofiles.open(
+            self.cacheFolderPath.joinpath("swiftshadow.pickle"), "wb+"
+        ) as cacheFile:
+            cacheExpiry = getExpiry(self.cachePeriod)
+            self.cacheExpiry = cacheExpiry
+            cache = CacheData(cacheExpiry, self.proxies)
+            pickled_bytes = dumps(cache)
+            _ = await cacheFile.write(pickled_bytes)
+        self.current = self.proxies[0]
 
     def update(self):
         """
@@ -135,6 +206,7 @@ class ProxyInterface:
                         f"Cache with {len(cache.proxies)} proxies, expire in {cache.expiryIn}"
                     )
                     self.current = self.proxies[0]
+                    logger.debug(f"Cache set to expire at {cache.expiryIn}")
                     self.cacheExpiry = cache.expiryIn
                     return
                 else:
@@ -142,6 +214,8 @@ class ProxyInterface:
         except FileNotFoundError:
             logger.info("No cache found, will be created after update.")
 
+        self.proxies = []
+
         for provider in Providers:
             if self.protocol not in provider.protocols:
                 continue
@@ -184,11 +258,13 @@ class ProxyInterface:
         Raises:
             ValueError: If validate_cache=True but no cache exists.
         """
-        if self.cacheExpiry and not validate_cache:
-            if checkExpiry(self.cacheExpiry):
-                self.update()
-        else:
-            raise ValueError("No cache availabel but validate_cache is true.")
+        if validate_cache:
+            if self.cacheExpiry:
+                if checkExpiry(self.cacheExpiry):
+                    logger.debug("Cache Expired on rotate call, updating.")
+                    self.update()
+            else:
+                raise ValueError("No cache available but validate_cache is true.")
         self.current = choice(self.proxies)
 
     def get(self) -> Proxy:
@@ -206,7 +282,7 @@ class ProxyInterface:
         """
 
         if self.autorotate:
-            self.rotate()
+            self.rotate(validate_cache=self.autoUpdate)
         if self.current:
             return self.current
         else:

swiftshadow/helpers.py

@@ -2,6 +2,7 @@ from datetime import datetime
 from typing import Literal
 
 from requests import get
+
 from swiftshadow.models import Proxy
 
 

swiftshadow/providers.py

@@ -3,7 +3,7 @@ from typing import Literal
 from requests import get
 
 from swiftshadow.helpers import plaintextToProxies
-from swiftshadow.models import Proxy, Provider
+from swiftshadow.models import Provider, Proxy
 from swiftshadow.types import MonosansProxyDict
 from swiftshadow.validator import validate_proxies
 

swiftshadow/validator.py

@@ -1,10 +1,47 @@
 import asyncio
+import re
 
 import aiohttp
 
 from swiftshadow.models import Proxy
 
 
+def find_ipv4_in_string(input_string: str) -> str | None:
+    """
+    Extract IPv4 Address from input and return the same.
+
+    Args:
+        input_string: Input string
+
+    Returns:
+        ipv4_address: If found
+    """
+    ipv4_pattern = r"\b((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\b"
+
+    match = re.search(ipv4_pattern, input_string)
+
+    if match:
+        return match.group(0)
+    else:
+        return None
+
+
+async def get_host_ip(async_session: aiohttp.ClientSession) -> str | None:
+    """
+    Gets the hosts external IP for validation.
+
+    Args:
+        async_session: AioHTTP client session object
+
+    Returns:
+        text: Host IP
+    """
+    async with async_session.get("http://checkip.amazonaws.com") as response:
+        text = await response.text()
+        ip = find_ipv4_in_string(text)
+        return ip
+
+
 async def check_proxy(async_session: aiohttp.ClientSession, proxy: Proxy) -> str:
     """
     Check one proxy abject.
@@ -38,13 +75,22 @@ async def validate_proxies(proxies: list[Proxy]) -> list[Proxy]:
     working_proxies: list[Proxy] = []
     async with aiohttp.ClientSession() as async_session:
         tasks = []
+
+        host_task = asyncio.create_task(coro=get_host_ip(async_session))
+        tasks.append(host_task)
+
         for proxy in proxies:
             task = asyncio.create_task(coro=check_proxy(async_session, proxy))
             tasks.append(task)
+
         results = await asyncio.gather(*tasks, return_exceptions=True)
+        host_ip = results[0]
+        results = results[1:]
+
         for proxy, result in zip(proxies, results):
             if type(result) is not str:
                 continue
-            if result.count(".") == 3:
+            result_ip = find_ipv4_in_string(result)
+            if result_ip and result_ip != host_ip:
                 working_proxies.append(proxy)
     return working_proxies

{swiftshadow-2.0.0.dist-info → swiftshadow-2.1.0.dist-info}/METADATA

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: swiftshadow
-Version: 2.0.0
+Version: 2.1.0
 Summary: Free IP Proxy rotator for python
 Author-email: sachin-sankar <mail.sachinsankar@gmail.com>
 License-File: LICENSE
 Requires-Python: >=3.12
+Requires-Dist: aiofiles>=24.1.0
 Requires-Dist: aiohttp>=3.11.11
 Requires-Dist: appdirs>=1.4.4
 Requires-Dist: requests>=2.32.3

swiftshadow-2.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+swiftshadow/__init__.py,sha256=oxJDmG0YO0XyYGQpa9V_o38lZnoie7U_d4Ik_h46UXs,922
+swiftshadow/cache.py,sha256=Mg8xsD6K3K012sILBwD2EZH6CE5kWCQNKCfZ5yadalI,800
+swiftshadow/classes.py,sha256=nTR_zykns2oxJ4Bj2UXxoXmLqNOMtkLqy17p_LH8odc,11423
+swiftshadow/exceptions.py,sha256=qu4eXyrkWD9qd4HCIR-8vRfVcqLlTupo4sD72alCdug,129
+swiftshadow/helpers.py,sha256=kC5PvfvDCQwigAmfkxlhb4PhcmgdCNpJksMiqyx9lU4,960
+swiftshadow/models.py,sha256=YyfZV98tPdLnF1O3WmTNUNoK4t0GuchfEftzjiM03ck,1678
+swiftshadow/providers.py,sha256=Z_NEOlMx0rr1Z5rfz-rA1kQ4YHKGRuI65jBxMGOyJkE,5660
+swiftshadow/types.py,sha256=Alyw3n54OESX1vSR-0kTvpYTlJ8LKfy5J9WZbtglHpE,894
+swiftshadow/validator.py,sha256=z0dmRKxhvPETSXfC2hTImL0t8pw0zjSYIhUaDMJcJhU,2469
+swiftshadow-2.1.0.dist-info/METADATA,sha256=NTPUMBc5xlCEkcryOEC81Au_MCD8JPO8QxrhkEDo7AM,3293
+swiftshadow-2.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+swiftshadow-2.1.0.dist-info/entry_points.txt,sha256=yMj0uEagcmXK2dmMmNXWebTpTT9j5K03oaRrd2wkyLA,49
+swiftshadow-2.1.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+swiftshadow-2.1.0.dist-info/RECORD,,

swiftshadow-2.0.0.dist-info/RECORD

@@ -1,14 +0,0 @@
-swiftshadow/__init__.py,sha256=DCxCxaMrluQDVJLyK5K61NxSaadD1d-nyTrFKsjfNDI,921
-swiftshadow/cache.py,sha256=eA_AWa8EsPdB6xD__ButvQdqETC4i89qEgxmHQV2XWU,800
-swiftshadow/classes.py,sha256=4hr0h2aOb-CXYPrf_ESf05RSEdRVLmYtvx09npnUVhQ,8242
-swiftshadow/exceptions.py,sha256=qu4eXyrkWD9qd4HCIR-8vRfVcqLlTupo4sD72alCdug,129
-swiftshadow/helpers.py,sha256=hHJ_JjRx2UFC5Ircl75LeYKBNDYTY_xMy2iWCk-UPqo,959
-swiftshadow/models.py,sha256=YyfZV98tPdLnF1O3WmTNUNoK4t0GuchfEftzjiM03ck,1678
-swiftshadow/providers.py,sha256=myJ6t-WD20wrjc7qDhxCpX1-oi7ipQutp34XKM2tjeI,5660
-swiftshadow/types.py,sha256=Alyw3n54OESX1vSR-0kTvpYTlJ8LKfy5J9WZbtglHpE,894
-swiftshadow/validator.py,sha256=dBcb6Lev6ojsk9f45IjyEUHZXsCcylyn6RoZA1WPDNQ,1345
-swiftshadow-2.0.0.dist-info/METADATA,sha256=MSM2WB68lsN__C1JiNwMH2sPMah2GrPUoCwyNbPoF9U,3261
-swiftshadow-2.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-swiftshadow-2.0.0.dist-info/entry_points.txt,sha256=yMj0uEagcmXK2dmMmNXWebTpTT9j5K03oaRrd2wkyLA,49
-swiftshadow-2.0.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-swiftshadow-2.0.0.dist-info/RECORD,,