cloudcheck 8.7.2__cp313-cp313-win_amd64.whl

cloudcheck/__init__.py

@@ -0,0 +1,3 @@
+from .cloudcheck import CloudCheck
+
+__all__ = ["CloudCheck"]

cloudcheck/helpers.py

@@ -0,0 +1,251 @@
+import ipaddress
+import os
+import httpx
+from pathlib import Path
+from typing import List, Set, Union
+
+
+def defrag_cidrs(
+    cidrs: List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]],
+) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
+    """
+    Defragment a list of CIDR blocks by merging adjacent networks.
+
+    Algorithm:
+    1. Sort by network bits (prefix length)
+    2. Iterate through pairs of adjacent networks
+    3. If networks have equal network bits and can be merged into a larger network,
+       replace them with the merged network
+    4. Repeat until no more merges are possible
+
+    Args:
+        cidrs: List of IPv4 or IPv6 network objects
+
+    Returns:
+        List of defragmented network objects
+    """
+    if not cidrs:
+        return []
+
+    # Convert to list and remove duplicates
+    networks = list(set(cidrs))
+
+    # Keep iterating until no more merges happen
+    changed = True
+    while changed:
+        changed = False
+
+        # Sort by network address
+        networks.sort(key=lambda x: (x.prefixlen, x.network_address.packed))
+
+        i = 0
+        while i < len(networks) - 1:
+            current = networks[i]
+            next_net = networks[i + 1]
+
+            # Check if we can merge these two networks
+            if _can_merge_networks(current, next_net):
+                # Create the merged network
+                merged = _merge_networks(current, next_net)
+
+                # Replace the two networks with the merged one
+                networks[i] = merged
+                networks.pop(i + 1)
+                changed = True
+            else:
+                i += 1
+
+    return networks
+
+
+def _can_merge_networks(
+    net1: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+    net2: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+) -> bool:
+    """
+    Check if two networks can be merged into a larger network.
+
+    Two networks can be merged if:
+    1. They have the same prefix length
+    2. They are adjacent (one starts where the other ends)
+    3. They can be combined into a network with prefix length - 1
+    """
+    # Must be same type (IPv4 or IPv6)
+    if net1.version != net2.version:
+        return False
+
+    # Must not be the same network
+    if net1 == net2:
+        return False
+
+    # Must have same prefix length
+    if net1.prefixlen != net2.prefixlen:
+        return False
+
+    # Must be adjacent networks
+    if not _are_adjacent_networks(net1, net2):
+        return False
+
+    return True
+
+
+def _are_adjacent_networks(
+    net1: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+    net2: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+) -> bool:
+    """
+    Check if two networks are adjacent by creating two networks with sub-1 CIDR
+    and checking if they are equal.
+    """
+    # Must have same prefix length
+    if net1.prefixlen != net2.prefixlen:
+        return False
+
+    # Create two networks with sub-1 CIDR
+    new_prefixlen = net1.prefixlen - 1
+    if new_prefixlen < 0:
+        return False
+
+    # Create the two networks with the reduced prefix length using supernet
+    net1_parent = net1.supernet(prefixlen_diff=1)
+    net2_parent = net2.supernet(prefixlen_diff=1)
+
+    # If they are equal, the networks are adjacent
+    return net1_parent == net2_parent
+
+
+def _merge_networks(
+    net1: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+    net2: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+) -> Union[ipaddress.IPv4Network, ipaddress.IPv6Network]:
+    """
+    Merge two adjacent networks into a larger network.
+    """
+    if net1 == net2:
+        raise ValueError("Networks must be different")
+
+    if not net1.version == net2.version:
+        raise ValueError("Networks must be the same version")
+
+    snet1 = net1.supernet(prefixlen_diff=1)
+    snet2 = net2.supernet(prefixlen_diff=1)
+    if not snet1 == snet2:
+        raise ValueError("Networks must be adjacent")
+
+    # Find the smaller network address
+    min_addr = min(net1.network_address, net2.network_address)
+
+    # Create the merged network with prefix length - 1
+    new_prefixlen = net1.prefixlen - 1
+    try:
+        return ipaddress.ip_network(f"{min_addr}/{new_prefixlen}")
+    except ValueError:
+        raise ValueError(
+            f"Failed to merge networks: {net1} (type: {type(net1)}) and {net2} (type: {type(net2)})"
+        )
+
+
+def cidrs_to_strings(
+    cidrs: List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]],
+) -> List[str]:
+    """
+    Convert a list of network objects to string representations.
+
+    Args:
+        cidrs: List of network objects
+
+    Returns:
+        List of CIDR strings
+    """
+    return [str(cidr) for cidr in cidrs]
+
+
+def strings_to_cidrs(
+    cidr_strings: List[str],
+) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
+    """
+    Convert a list of CIDR strings to network objects.
+
+    Args:
+        cidr_strings: List of CIDR strings
+
+    Returns:
+        List of network objects
+    """
+    networks = []
+    for cidr_str in cidr_strings:
+        try:
+            networks.append(ipaddress.ip_network(cidr_str, strict=False))
+        except ValueError:
+            # Skip invalid CIDR strings
+            continue
+    return networks
+
+
+browser_base_headers = {
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "en-US,en;q=0.9",
+    "cache-control": "no-cache",
+    "pragma": "no-cache",
+    "priority": "u=0, i",
+    "referer": "https://www.google.com/",
+    "sec-ch-ua": '"Chromium";v="127", "Not)A;Brand";v="99"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"Linux"',
+    "sec-fetch-dest": "document",
+    "sec-fetch-mode": "navigate",
+    "sec-fetch-site": "cross-site",
+    "sec-fetch-user": "?1",
+    "upgrade-insecure-requests": "1",
+    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+}
+
+
+def request(url, include_api_key=False, browser_headers=False, timeout=60, **kwargs):
+    headers = kwargs.get("headers", {})
+    if browser_headers:
+        headers.update(browser_base_headers)
+    bbot_io_api_key = os.getenv("BBOT_IO_API_KEY")
+    if include_api_key and bbot_io_api_key:
+        headers["Authorization"] = f"Bearer {bbot_io_api_key}"
+    kwargs["headers"] = headers
+    kwargs["timeout"] = timeout
+    kwargs.setdefault("follow_redirects", True)
+    return httpx.get(url, **kwargs)
+
+
+def parse_v2fly_domain_file(file_path: Path) -> Set[str]:
+    """Parse a domain list file and extract domains."""
+    print(f"Parsing {file_path}")
+    domains = set()
+    if not file_path.exists():
+        print(f"File {file_path} does not exist")
+        return domains
+
+    with open(file_path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            # Handle inline comments by splitting on # and taking the first part
+            line = line.split("#")[0].strip()
+            if not line:
+                continue
+
+            if line.startswith("include:"):
+                include_file = line[8:]
+                include_path = file_path.parent / include_file
+                domains.update(parse_v2fly_domain_file(include_path))
+                continue
+
+            if line.startswith("domain:"):
+                domain = line[7:]
+            elif line.startswith("full:"):
+                domain = line[5:]
+            elif line.startswith("keyword:") or line.startswith("regexp:"):
+                continue
+            else:
+                domain = line
+
+            domain = domain.split("@")[0].strip()
+            if domain:
+                domains.add(domain.lower())
+    return domains

cloudcheck/providers/__init__.py

@@ -0,0 +1,51 @@
+import importlib
+from sys import stderr
+from pathlib import Path
+from typing import Dict, Type
+
+from cloudcheck.providers.base import BaseProvider
+
+# Dictionary to store loaded provider classes
+_provider_classes: Dict[str, Type[BaseProvider]] = {}
+_provider_instances: Dict[str, BaseProvider] = {}
+
+
+def load_provider_classes() -> Dict[str, Type[BaseProvider]]:
+    """Dynamically load all cloud provider classes from the providers directory."""
+    global _provider_classes
+
+    if _provider_classes:
+        return _provider_classes
+
+    providers_path = Path(__file__).parent
+
+    for file in providers_path.glob("*.py"):
+        if file.stem in ("base", "__init__"):
+            continue
+
+        try:
+            import_path = f"cloudcheck.providers.{file.stem}"
+            module = importlib.import_module(import_path)
+
+            # Look for classes that inherit from BaseProvider
+            for attr_name in dir(module):
+                attr = getattr(module, attr_name)
+                if (
+                    isinstance(attr, type)
+                    and issubclass(attr, BaseProvider)
+                    and attr != BaseProvider
+                ):
+                    provider_name = attr.__name__
+                    _provider_classes[provider_name] = attr
+
+        except Exception as e:
+            print(f"Failed to load provider from {file}: {e}", file=stderr)
+            raise
+
+    return _provider_classes
+
+
+for provider_name, provider_class in load_provider_classes().items():
+    provider_instance = provider_class()
+    globals()[provider_name] = provider_instance
+    _provider_instances[provider_name] = provider_instance

cloudcheck/providers/akamai.py

@@ -0,0 +1,35 @@
+import io
+import zipfile
+from cloudcheck.providers.base import BaseProvider
+from typing import List
+
+
+class Akamai(BaseProvider):
+    v2fly_company: str = "akamai"
+    tags: List[str] = ["cloud"]
+    short_description: str = "Akamai"
+    long_description: str = "A content delivery network and cloud services provider that delivers web and internet security services."
+    # {"org_id": "AKAMAI-ARIN", "org_name": "Akamai Technologies, Inc.", "country": "US", "asns": [12222,16625,16702,17204,17334,18680,18717,20189,22207,22452,23454,23455,26008,30675,31984,32787,33047,35993,35994,36029,36183,393234,393560]}
+    # {"org_id": "ORG-AT1-RIPE", "org_name": "Akamai International B.V.", "country": "NL", "asns": [20940,21342,21357,21399,31107,31108,31109,31110,31377,33905,34164,34850,35204,39836,43639,48163,49249,49846,200005,213120]}
+    # {"org_id": "ORG-ATI1-AP-APNIC", "org_name": "Akamai Technologies, Inc.", "country": "US", "asns": [23903,24319,45757,55409,55770,63949,133103]}
+    org_ids: List[str] = [
+        "AKAMAI-ARIN",
+        "ORG-AT1-RIPE",
+        "ORG-ATI1-AP-APNIC",
+    ]
+
+    _ips_url = "https://techdocs.akamai.com/property-manager/pdfs/akamai_ipv4_ipv6_CIDRs-txt.zip"
+
+    def fetch_cidrs(self):
+        response = self.request(self._ips_url)
+        ranges = set()
+        content = getattr(response, "content", b"")
+        # Extract the contents of the zip file to memory
+        with zipfile.ZipFile(io.BytesIO(content)) as zip_file:
+            for filename in ("akamai_ipv4_CIDRs.txt", "akamai_ipv6_CIDRs.txt"):
+                with zip_file.open(filename) as f:
+                    for line in f.read().splitlines():
+                        line = line.decode(errors="ignore").strip()
+                        if line:
+                            ranges.add(line)
+        return list(ranges)

cloudcheck/providers/alibaba.py

@@ -0,0 +1,13 @@
+from cloudcheck.providers.base import BaseProvider
+from typing import List
+
+
+class Alibaba(BaseProvider):
+    v2fly_company: str = "alibaba"
+    tags: List[str] = ["cloud"]
+    short_description: str = "Alibaba Cloud"
+    long_description: str = "A Chinese cloud computing company and subsidiary of Alibaba Group, providing cloud services and infrastructure."
+    # {"org_id": "ORG-ASEP1-AP-APNIC", "org_name": "Alibaba Cloud (Singapore) Private Limited", "country": "SG", "asns": [134963]}
+    org_ids: List[str] = [
+        "ORG-ASEP1-AP-APNIC",
+    ]

cloudcheck/providers/amazon.py

@@ -0,0 +1,39 @@
+from cloudcheck.providers.base import BaseProvider
+from typing import List, Dict
+
+
+class Amazon(BaseProvider):
+    v2fly_company: str = "amazon"
+    short_description: str = "Amazon Web Services"
+    long_description: str = "A comprehensive cloud computing platform provided by Amazon, offering infrastructure services, storage, and computing power."
+    org_ids: List[str] = [
+        "AMAZO-139-ARIN",  # Amazon.com, Inc., US
+        "AMAZO-141-ARIN",  # Amazon Technologies, Inc., US
+        "AMAZO-22-ARIN",  # Amazon Web Services, Inc., US
+        "AMAZO-4-ARIN",  # Amazon.com, Inc., US
+        "AMAZON-4-ARIN",  # Amazon.com, Inc., US
+        "ARL-76-ARIN",  # Amazon Robotics LLC, US
+        "ASL-830-ARIN",  # Amazon.com Services, LLC, US
+        "AT-9049-ARIN",  # Amazon Technologies Inc., US
+        "AT-9066-ARIN",  # Amazon Technologies Inc., US
+        "ORG-AARP1-AP-APNIC",  # Amazon Asia-Pacific Resources Private Limited, SG
+        "ORG-ACSP2-AP-APNIC",  # Amazon Corporate Services Pty Ltd, AU
+        "ORG-ACTS1-AP-APNIC",  # Amazon Connection Technology Services (Beijing) Co., LTD, CN
+        "ORG-ADSI1-RIPE",  # Amazon Data Services Ireland Ltd, IE
+        "ORG-ADSJ1-AP-APNIC",  # Amazon Data Services Japan KK, JP
+        "ORG-AI2-AP-APNIC",  # Amazon.com, Inc., US
+    ]
+    tags: List[str] = ["cloud"]
+    _bucket_name_regex = r"[a-z0-9_][a-z0-9-\.]{1,61}[a-z0-9]"
+    regexes: Dict[str, List[str]] = {
+        "STORAGE_BUCKET_NAME": [_bucket_name_regex],
+        "STORAGE_BUCKET_HOSTNAME": [
+            r"(" + _bucket_name_regex + r")\.(s3-?(?:[a-z0-9-]*\.){1,2}amazonaws\.com)"
+        ],
+    }
+
+    _ips_url = "https://ip-ranges.amazonaws.com/ip-ranges.json"
+
+    def fetch_cidrs(self):
+        response = self.request(self._ips_url)
+        return list(set(p["ip_prefix"] for p in response.json()["prefixes"]))

cloudcheck/providers/arvancloud.py

@@ -0,0 +1,22 @@
+from cloudcheck.providers.base import BaseProvider
+from typing import List
+
+
+class Arvancloud(BaseProvider):
+    domains: List[str] = ["arvancloud.ir"]
+    tags: List[str] = ["cdn"]
+    short_description: str = "Arvancloud"
+    long_description: str = "An Iranian cloud computing and content delivery network provider offering cloud infrastructure and CDN services."
+    # {"org_id": "ORG-AGTL2-RIPE", "org_name": "ARVANCLOUD GLOBAL TECHNOLOGIES L.L.C", "country": "AE", "asns": [57568,208006,210296]}
+    org_ids: List[str] = [
+        "ORG-AGTL2-RIPE",
+    ]
+
+    _ips_url = "https://www.arvancloud.ir/en/ips.txt"
+
+    def fetch_cidrs(self):
+        response = self.request(self._ips_url)
+        ranges = set()
+        if getattr(response, "status_code", 0) == 200:
+            ranges.update(response.text.splitlines())
+        return list(ranges)

cloudcheck/providers/backblaze.py

@@ -0,0 +1,13 @@
+from cloudcheck.providers.base import BaseProvider
+from typing import List
+
+
+class Backblaze(BaseProvider):
+    tags: List[str] = ["cloud"]
+    short_description: str = "Backblaze"
+    long_description: str = "A cloud storage and backup service provider offering data backup and cloud storage solutions."
+    # {"org_id": "BACKB-7-ARIN", "org_name": "Backblaze Inc", "country": "US", "asns": [40401,396865]}
+    org_ids: List[str] = [
+        "BACKB-7-ARIN",
+    ]
+    domains: List[str] = ["backblaze.com", "backblazeb2.com"]

cloudcheck/providers/baidu.py

@@ -0,0 +1,17 @@
+from cloudcheck.providers.base import BaseProvider
+from typing import List
+
+
+class Baidu(BaseProvider):
+    v2fly_company: str = "baidu"
+    tags: List[str] = ["cdn"]
+    short_description: str = "Baidu Cloud Acceleration (百度云加速)"
+    long_description: str = "A Chinese content delivery network and cloud acceleration service provided by Baidu."
+    # {"org_id": "BUL-5-ARIN", "org_name": "Baidu USA LLC", "country": "US", "asns": [63288]}
+    # {"org_id": "ORG-BKL1-AP-APNIC", "org_name": "Baidu (Hong Kong) Limited", "country": "HK", "asns": [133746]}
+    # {"org_id": "ORG-BKL4-RIPE", "org_name": "Baidu (Hong Kong) Limited", "country": "HK", "asns": [199506]}
+    org_ids: List[str] = [
+        "BUL-5-ARIN",
+        "ORG-BKL1-AP-APNIC",
+        "ORG-BKL4-RIPE",
+    ]