cloudcheck 8.4.0__cp312-cp312-win_amd64.whl

cloudcheck/__init__.py

@@ -0,0 +1,3 @@
+from .cloudcheck import CloudCheck
+
+__all__ = ["CloudCheck"]

cloudcheck/helpers.py

@@ -0,0 +1,211 @@
+import ipaddress
+import os
+import requests
+from typing import List, Union
+
+
+def defrag_cidrs(
+    cidrs: List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]],
+) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
+    """
+    Defragment a list of CIDR blocks by merging adjacent networks.
+
+    Algorithm:
+    1. Sort by network bits (prefix length)
+    2. Iterate through pairs of adjacent networks
+    3. If networks have equal network bits and can be merged into a larger network,
+       replace them with the merged network
+    4. Repeat until no more merges are possible
+
+    Args:
+        cidrs: List of IPv4 or IPv6 network objects
+
+    Returns:
+        List of defragmented network objects
+    """
+    if not cidrs:
+        return []
+
+    # Convert to list and remove duplicates
+    networks = list(set(cidrs))
+
+    # Keep iterating until no more merges happen
+    changed = True
+    while changed:
+        changed = False
+
+        # Sort by network address
+        networks.sort(key=lambda x: (x.prefixlen, x.network_address.packed))
+
+        i = 0
+        while i < len(networks) - 1:
+            current = networks[i]
+            next_net = networks[i + 1]
+
+            # Check if we can merge these two networks
+            if _can_merge_networks(current, next_net):
+                # Create the merged network
+                merged = _merge_networks(current, next_net)
+
+                # Replace the two networks with the merged one
+                networks[i] = merged
+                networks.pop(i + 1)
+                changed = True
+            else:
+                i += 1
+
+    return networks
+
+
+def _can_merge_networks(
+    net1: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+    net2: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+) -> bool:
+    """
+    Check if two networks can be merged into a larger network.
+
+    Two networks can be merged if:
+    1. They have the same prefix length
+    2. They are adjacent (one starts where the other ends)
+    3. They can be combined into a network with prefix length - 1
+    """
+    # Must be same type (IPv4 or IPv6)
+    if net1.version != net2.version:
+        return False
+
+    # Must not be the same network
+    if net1 == net2:
+        return False
+
+    # Must have same prefix length
+    if net1.prefixlen != net2.prefixlen:
+        return False
+
+    # Must be adjacent networks
+    if not _are_adjacent_networks(net1, net2):
+        return False
+
+    return True
+
+
+def _are_adjacent_networks(
+    net1: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+    net2: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+) -> bool:
+    """
+    Check if two networks are adjacent by creating two networks with sub-1 CIDR
+    and checking if they are equal.
+    """
+    # Must have same prefix length
+    if net1.prefixlen != net2.prefixlen:
+        return False
+
+    # Create two networks with sub-1 CIDR
+    new_prefixlen = net1.prefixlen - 1
+    if new_prefixlen < 0:
+        return False
+
+    # Create the two networks with the reduced prefix length using supernet
+    net1_parent = net1.supernet(prefixlen_diff=1)
+    net2_parent = net2.supernet(prefixlen_diff=1)
+
+    # If they are equal, the networks are adjacent
+    return net1_parent == net2_parent
+
+
+def _merge_networks(
+    net1: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+    net2: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
+) -> Union[ipaddress.IPv4Network, ipaddress.IPv6Network]:
+    """
+    Merge two adjacent networks into a larger network.
+    """
+    if net1 == net2:
+        raise ValueError("Networks must be different")
+
+    if not net1.version == net2.version:
+        raise ValueError("Networks must be the same version")
+
+    snet1 = net1.supernet(prefixlen_diff=1)
+    snet2 = net2.supernet(prefixlen_diff=1)
+    if not snet1 == snet2:
+        raise ValueError("Networks must be adjacent")
+
+    # Find the smaller network address
+    min_addr = min(net1.network_address, net2.network_address)
+
+    # Create the merged network with prefix length - 1
+    new_prefixlen = net1.prefixlen - 1
+    try:
+        return ipaddress.ip_network(f"{min_addr}/{new_prefixlen}")
+    except ValueError:
+        raise ValueError(
+            f"Failed to merge networks: {net1} (type: {type(net1)}) and {net2} (type: {type(net2)})"
+        )
+
+
+def cidrs_to_strings(
+    cidrs: List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]],
+) -> List[str]:
+    """
+    Convert a list of network objects to string representations.
+
+    Args:
+        cidrs: List of network objects
+
+    Returns:
+        List of CIDR strings
+    """
+    return [str(cidr) for cidr in cidrs]
+
+
+def strings_to_cidrs(
+    cidr_strings: List[str],
+) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
+    """
+    Convert a list of CIDR strings to network objects.
+
+    Args:
+        cidr_strings: List of CIDR strings
+
+    Returns:
+        List of network objects
+    """
+    networks = []
+    for cidr_str in cidr_strings:
+        try:
+            networks.append(ipaddress.ip_network(cidr_str, strict=False))
+        except ValueError:
+            # Skip invalid CIDR strings
+            continue
+    return networks
+
+
+browser_base_headers = {
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "en-US,en;q=0.9",
+    "cache-control": "no-cache",
+    "pragma": "no-cache",
+    "priority": "u=0, i",
+    "referer": "https://www.google.com/",
+    "sec-ch-ua": '"Chromium";v="127", "Not)A;Brand";v="99"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"Linux"',
+    "sec-fetch-dest": "document",
+    "sec-fetch-mode": "navigate",
+    "sec-fetch-site": "cross-site",
+    "sec-fetch-user": "?1",
+    "upgrade-insecure-requests": "1",
+    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+}
+
+
+def request(url, include_api_key=False, browser_headers=False, **kwargs):
+    headers = kwargs.get("headers", {})
+    if browser_headers:
+        headers.update(browser_base_headers)
+    bbot_io_api_key = os.getenv("BBOT_IO_API_KEY")
+    if include_api_key and bbot_io_api_key:
+        headers["Authorization"] = f"Bearer {bbot_io_api_key}"
+    kwargs["headers"] = headers
+    return requests.get(url, **kwargs)

cloudcheck/providers/akamai.py

@@ -0,0 +1,33 @@
+import io
+import zipfile
+from cloudcheck.providers.base import BaseProvider
+from typing import List
+
+
+class Akamai(BaseProvider):
+    v2fly_company: str = "akamai"
+    tags: List[str] = ["cloud"]
+    # {"org_id": "AKAMAI-ARIN", "org_name": "Akamai Technologies, Inc.", "country": "US", "asns": [12222,16625,16702,17204,17334,18680,18717,20189,22207,22452,23454,23455,26008,30675,31984,32787,33047,35993,35994,36029,36183,393234,393560]}
+    # {"org_id": "ORG-AT1-RIPE", "org_name": "Akamai International B.V.", "country": "NL", "asns": [20940,21342,21357,21399,31107,31108,31109,31110,31377,33905,34164,34850,35204,39836,43639,48163,49249,49846,200005,213120]}
+    # {"org_id": "ORG-ATI1-AP-APNIC", "org_name": "Akamai Technologies, Inc.", "country": "US", "asns": [23903,24319,45757,55409,55770,63949,133103]}
+    org_ids: List[str] = [
+        "AKAMAI-ARIN",
+        "ORG-AT1-RIPE",
+        "ORG-ATI1-AP-APNIC",
+    ]
+
+    _ips_url = "https://techdocs.akamai.com/property-manager/pdfs/akamai_ipv4_ipv6_CIDRs-txt.zip"
+
+    def fetch_cidrs(self):
+        response = self.request(self._ips_url)
+        ranges = set()
+        content = getattr(response, "content", b"")
+        # Extract the contents of the zip file to memory
+        with zipfile.ZipFile(io.BytesIO(content)) as zip_file:
+            for filename in ("akamai_ipv4_CIDRs.txt", "akamai_ipv6_CIDRs.txt"):
+                with zip_file.open(filename) as f:
+                    for line in f.read().splitlines():
+                        line = line.decode(errors="ignore").strip()
+                        if line:
+                            ranges.add(line)
+        return list(ranges)

cloudcheck/providers/alibaba.py

@@ -0,0 +1,11 @@
+from cloudcheck.providers.base import BaseProvider
+from typing import List
+
+
+class Alibaba(BaseProvider):
+    v2fly_company: str = "alibaba"
+    tags: List[str] = ["cloud"]
+    # {"org_id": "ORG-ASEP1-AP-APNIC", "org_name": "Alibaba Cloud (Singapore) Private Limited", "country": "SG", "asns": [134963]}
+    org_ids: List[str] = [
+        "ORG-ASEP1-AP-APNIC",
+    ]

cloudcheck/providers/amazon.py

@@ -0,0 +1,37 @@
+from cloudcheck.providers.base import BaseProvider
+from typing import List, Dict
+
+
+class Amazon(BaseProvider):
+    v2fly_company: str = "amazon"
+    org_ids: List[str] = [
+        "AMAZO-139-ARIN",  # Amazon.com, Inc., US
+        "AMAZO-141-ARIN",  # Amazon Technologies, Inc., US
+        "AMAZO-22-ARIN",  # Amazon Web Services, Inc., US
+        "AMAZO-4-ARIN",  # Amazon.com, Inc., US
+        "AMAZON-4-ARIN",  # Amazon.com, Inc., US
+        "ARL-76-ARIN",  # Amazon Robotics LLC, US
+        "ASL-830-ARIN",  # Amazon.com Services, LLC, US
+        "AT-9049-ARIN",  # Amazon Technologies Inc., US
+        "AT-9066-ARIN",  # Amazon Technologies Inc., US
+        "ORG-AARP1-AP-APNIC",  # Amazon Asia-Pacific Resources Private Limited, SG
+        "ORG-ACSP2-AP-APNIC",  # Amazon Corporate Services Pty Ltd, AU
+        "ORG-ACTS1-AP-APNIC",  # Amazon Connection Technology Services (Beijing) Co., LTD, CN
+        "ORG-ADSI1-RIPE",  # Amazon Data Services Ireland Ltd, IE
+        "ORG-ADSJ1-AP-APNIC",  # Amazon Data Services Japan KK, JP
+        "ORG-AI2-AP-APNIC",  # Amazon.com, Inc., US
+    ]
+    tags: List[str] = ["cloud"]
+    _bucket_name_regex = r"[a-z0-9_][a-z0-9-\.]{1,61}[a-z0-9]"
+    regexes: Dict[str, List[str]] = {
+        "STORAGE_BUCKET_NAME": [_bucket_name_regex],
+        "STORAGE_BUCKET_HOSTNAME": [
+            r"(" + _bucket_name_regex + r")\.(s3-?(?:[a-z0-9-]*\.){1,2}amazonaws\.com)"
+        ],
+    }
+
+    _ips_url = "https://ip-ranges.amazonaws.com/ip-ranges.json"
+
+    def fetch_cidrs(self):
+        response = self.request(self._ips_url)
+        return list(set(p["ip_prefix"] for p in response.json()["prefixes"]))

cloudcheck/providers/arvancloud.py

@@ -0,0 +1,20 @@
+from cloudcheck.providers.base import BaseProvider
+from typing import List
+
+
+class Arvancloud(BaseProvider):
+    domains: List[str] = ["arvancloud.ir"]
+    tags: List[str] = ["cdn"]
+    # {"org_id": "ORG-AGTL2-RIPE", "org_name": "ARVANCLOUD GLOBAL TECHNOLOGIES L.L.C", "country": "AE", "asns": [57568,208006,210296]}
+    org_ids: List[str] = [
+        "ORG-AGTL2-RIPE",
+    ]
+
+    _ips_url = "https://www.arvancloud.ir/en/ips.txt"
+
+    def fetch_cidrs(self):
+        response = self.request(self._ips_url)
+        ranges = set()
+        if getattr(response, "status_code", 0) == 200:
+            ranges.update(response.text.splitlines())
+        return list(ranges)

cloudcheck/providers/azure.py

@@ -0,0 +1,37 @@
+from cloudcheck.providers.base import BaseProvider
+from typing import List, Dict
+
+
+class Azure(BaseProvider):
+    v2fly_company: str = "azure"
+    tags: List[str] = ["cloud"]
+    # {"org_id": "MSFT-ARIN", "org_name": "Microsoft Corporation", "country": "US", "asns": [3598,5761,6182,6194,6291,6584,8068,8069,8070,8071,8072,8073,8074,8075,12076,13399,13811,14719,14783,17144,17345,20046,22692,23468,25796,26222,30135,30520,30575,31792,32476,36006,40066,46182,54396,63245,63314,395496,395524,395851,396463,397096,397466,397996,398575,398656,398657,398658,398659,398660,398661,398961,400572,400573,400574,400575,400576,400577,400578,400579,400580,400581,400582,400884]}
+    # {"org_id": "ORG-MA42-RIPE", "org_name": "Microsoft Limited", "country": "GB", "asns": [35106]}
+    # {"org_id": "ORG-MDMG3-RIPE", "org_name": "Microsoft Deutschland MCIO GmbH", "country": "DE", "asns": [200517]}
+    # {"org_id": "ORG-MOPL2-AP-APNIC", "org_name": "Microsoft Operations PTE Ltd", "country": "SG", "asns": [132348]}
+    # {"org_id": "ORG-MSPL4-AP-APNIC", "org_name": "Microsoft Singapore Pte. Ltd.", "country": "US", "asns": [45139]}
+    org_ids: List[str] = [
+        "MSFT-ARIN",
+        "ORG-MA42-RIPE",
+        "ORG-MDMG3-RIPE",
+        "ORG-MOPL2-AP-APNIC",
+        "ORG-MSPL4-AP-APNIC",
+    ]
+    _bucket_name_regex = r"[a-z0-9][a-z0-9-_\.]{1,61}[a-z0-9]"
+    regexes: Dict[str, List[str]] = {
+        "STORAGE_BUCKET_NAME": [_bucket_name_regex],
+        "STORAGE_BUCKET_HOSTNAME": [
+            r"(" + _bucket_name_regex + r")\.(blob\.core\.windows\.net)"
+        ],
+    }
+
+    _ips_url = "https://download.microsoft.com/download/0/1/8/018E208D-54F8-44CD-AA26-CD7BC9524A8C/PublicIPs_20200824.xml"
+
+    def fetch_cidrs(self):
+        response = self.request(self._ips_url)
+        ranges = set()
+        for line in response.text.splitlines():
+            if "IpRange Subnet" in line:
+                ip_range = line.split('"')[1]
+                ranges.add(ip_range)
+        return list(ranges)

cloudcheck/providers/backblaze.py

@@ -0,0 +1,11 @@
+from cloudcheck.providers.base import BaseProvider
+from typing import List
+
+
+class Backblaze(BaseProvider):
+    tags: List[str] = ["cloud"]
+    # {"org_id": "BACKB-7-ARIN", "org_name": "Backblaze Inc", "country": "US", "asns": [40401,396865]}
+    org_ids: List[str] = [
+        "BACKB-7-ARIN",
+    ]
+    domains: List[str] = ["backblaze.com", "backblazeb2.com"]

cloudcheck/providers/base.py

@@ -0,0 +1,303 @@
+import ipaddress
+import os
+import traceback
+import subprocess
+import time
+from pathlib import Path
+from typing import Dict, List, Union, Set
+from pydantic import BaseModel, field_validator, computed_field
+
+from ..helpers import defrag_cidrs, request
+
+
+v2fly_repo_pulled = False
+
+
+class BaseProvider(BaseModel):
+    """
+    Base class for all cloud providers.
+
+    Each provider inherits from this class and overrides any of the default values.
+    They can also override the update_cidrs() method to fetch cidrs from a different source.
+    """
+
+    # these values are static and always loaded from the class definition
+    regexes: Dict[str, List[str]] = {}
+    tags: List[str] = []  # Tags for the provider (e.g. "cdn", "waf", etc.)
+    org_ids: List[str] = []  # ASN Organization IDs (e.g. GOGL-ARIN)
+    v2fly_company: str = ""  # Company name for v2fly domain fetching
+
+    # these values are dynamic and set by the update() method
+    last_updated: float = time.time()
+
+    # these we allow static values but they are later merged with dynamic values
+    asns: List[int] = []
+    cidrs: List[str] = []
+    domains: List[str] = []
+
+    @field_validator("cidrs")
+    @classmethod
+    def validate_cidrs(cls, value):
+        ips = []
+        for v in value:
+            try:
+                ips.append(ipaddress.ip_network(v, strict=False))
+            except ValueError:
+                print(f"Invalid CIDR: from {cls.__name__}: {v}")
+                continue
+        ips = [str(ip) for ip in defrag_cidrs(ips)]
+        return sorted(ips)
+
+    @field_validator("domains")
+    @classmethod
+    def validate_domains(cls, value):
+        return sorted(list(set([d.lower().strip(".") for d in value])))
+
+    @computed_field(return_type=str)
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        print(f"Initializing {self.name}")
+        self._cidrs = []
+        self._cache_dir = Path.home() / ".cache" / "cloudcheck"
+        self._repo_url = "https://github.com/v2fly/domain-list-community.git"
+        self._asndb_url = os.getenv("ASNDB_URL", "https://asndb.api.bbot.io/v1")
+        self._bbot_io_api_key = os.getenv("BBOT_IO_API_KEY")
+
+    def update(self):
+        print(f"Updating {self.name}")
+        errors = []
+        errors.extend(self.update_domains())
+        errors.extend(self.update_cidrs())
+        return errors
+
+    def update_domains(self):
+        # update dynamic domains
+        errors = []
+        if self.v2fly_company:
+            domains, errors = self.fetch_v2fly_domains()
+            if domains:
+                self.domains = sorted(list(set(self.domains + domains)))
+            else:
+                errors.append(
+                    f"No v2fly domains were found for {self.name} (company name: {self.v2fly_company})"
+                )
+        return errors
+
+    def update_cidrs(self):
+        cidrs = set()
+        errors = []
+
+        # query by direct ASNs
+        if self.asns:
+            _cidrs, _errors = self.fetch_asns()
+            print(f"Got {len(_cidrs)} ASN cidrs for {self.name}'s ASNs {self.asns}")
+            if not _cidrs:
+                errors.append(
+                    f"No ASN cidrs were found for {self.name}'s ASNs {self.asns}"
+                )
+            errors.extend(_errors)
+            cidrs.update(_cidrs)
+
+        # query by org IDs
+        if self.org_ids:
+            _cidrs, _asns, _errors = self.fetch_org_ids()
+            _asns = _asns.copy()
+            _asns.update(self.asns)
+            self.asns = list(sorted(_asns))
+            print(
+                f"Got {len(_cidrs)} org id cidrs for {self.name}'s org ids {self.org_ids}"
+            )
+            if not _cidrs:
+                errors.append(
+                    f"No cidrs were found for {self.name}'s org ids {self.org_ids}"
+                )
+            errors.extend(_errors)
+            cidrs.update(_cidrs)
+
+        # fetch any dynamically-updated lists of CIDRs
+        try:
+            dynamic_cidrs = self.fetch_cidrs()
+            print(f"Got {len(dynamic_cidrs)} dynamic cidrs for {self.name}")
+            cidrs.update(dynamic_cidrs)
+        except Exception as e:
+            errors.append(
+                f"Failed to fetch dynamic cidrs for {self.name}: {e}:\n{traceback.format_exc()}"
+            )
+
+        # finally, put in any manually-specified CIDRs
+        print(f"Adding {len(self.cidrs)} manually-specified cidrs for {self.name}")
+        if self.cidrs:
+            cidrs.update(self.cidrs)
+
+        print(f"Total {len(cidrs)} cidrs for {self.name}")
+
+        try:
+            self.cidrs = self.validate_cidrs(cidrs)
+        except Exception as e:
+            errors.append(
+                f"Error validating ASN cidrs for {self.name}: {e}:\n{traceback.format_exc()}"
+            )
+
+        self.last_updated = time.time()
+
+        return errors
+
+    def fetch_org_ids(
+        self,
+    ) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
+        """Takes org_ids and populates the .asns and .cidrs attributes."""
+        errors = []
+        cidrs = set()
+        print(f"Fetching {len(self.org_ids)} org ids for {self.name}")
+        asns = set()
+        for org_id in self.org_ids:
+            print(f"Fetching cidrs for {org_id} from asndb")
+            try:
+                url = f"{self._asndb_url}/org/{org_id}"
+                print(f"Fetching {url}")
+                res = self.request(url, include_api_key=True)
+                print(f"{url} -> {res}: {res.text}")
+                j = res.json()
+            except Exception as e:
+                errors.append(
+                    f"Failed to fetch cidrs for {org_id} from asndb: {e}:\n{traceback.format_exc()}"
+                )
+                continue
+            _asns = j.get("asns", [])
+            for asn in _asns:
+                asns.add(asn)
+                asn_cidrs, _errors = self.fetch_asn(asn)
+                errors.extend(_errors)
+                cidrs.update(asn_cidrs)
+        return cidrs, asns, errors
+
+    def fetch_asns(self) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
+        """Fetch CIDRs for a given list of ASNs from ASNDB."""
+        cidrs = []
+        errors = []
+        print(f"Fetching {len(self.asns)} ASNs for {self.name}")
+        for asn in self.asns:
+            asn_cidrs, _errors = self.fetch_asn(asn)
+            errors.extend(_errors)
+            cidrs.update(asn_cidrs)
+        return cidrs, errors
+
+    def fetch_asn(
+        self, asn: int
+    ) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
+        """Fetch CIDRs for a given ASN from ASNDB."""
+        cidrs = []
+        errors = []
+        url = f"{self._asndb_url}/asn/{asn}"
+        print(f"Fetching {url}")
+        try:
+            res = self.request(url, include_api_key=True)
+            print(f"{url} -> {res.text}")
+            j = res.json()
+            cidrs = j.get("subnets", [])
+        except Exception as e:
+            errors.append(
+                f"Failed to fetch cidrs for {asn} from asndb: {e}:\n{traceback.format_exc()}"
+            )
+        print(f"Got {len(cidrs)} cidrs for {asn}")
+        return cidrs, errors
+
+    def fetch_v2fly_domains(self) -> List[str]:
+        """Fetch domains from the v2fly community repository."""
+        if not self.v2fly_company:
+            return [], []
+
+        errors = []
+        repo_path, _success = self._ensure_v2fly_repo_cached()
+        company_file = repo_path / "data" / self.v2fly_company
+        try:
+            domains = self._parse_v2fly_domain_file(company_file)
+        except Exception as e:
+            errors.append(
+                f"Failed to parse {self.v2fly_company} domains: {e}:\n{traceback.format_exc()}"
+            )
+        return sorted(list(domains)), errors
+
+    def fetch_cidrs(self) -> List[str]:
+        """Fetch CIDRs from a custom source."""
+        return []
+
+    def fetch_domains(self) -> List[str]:
+        """Fetch domains from a custom source."""
+        return []
+
+    def _ensure_v2fly_repo_cached(self) -> Path:
+        """Ensure the community repo is cloned and up-to-date."""
+        global v2fly_repo_pulled
+        errors = []
+        repo_dir = self._cache_dir / "domain-list-community"
+        if not repo_dir.exists():
+            self._cache_dir.mkdir(parents=True, exist_ok=True)
+            try:
+                subprocess.run(
+                    ["git", "clone", "--depth", "1", self._repo_url, str(repo_dir)],
+                    check=True,
+                    capture_output=True,
+                )
+                v2fly_repo_pulled = True
+            except subprocess.CalledProcessError as e:
+                errors.append(
+                    f"Failed to clone v2fly repo: {e}:\n{traceback.format_exc()}"
+                )
+        elif not v2fly_repo_pulled:
+            try:
+                subprocess.run(
+                    ["git", "pull"], cwd=repo_dir, check=True, capture_output=True
+                )
+            except subprocess.CalledProcessError as e:
+                errors.append(
+                    f"Failed to pull v2fly repo: {e}:\n{traceback.format_exc()}"
+                )
+        return repo_dir, errors
+
+    def _parse_v2fly_domain_file(self, file_path: Path) -> Set[str]:
+        """Parse a domain list file and extract domains."""
+        print(f"Parsing {file_path}")
+        domains = set()
+        if not file_path.exists():
+            print(f"File {file_path} does not exist")
+            return domains
+
+        with open(file_path, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+
+                if line.startswith("include:"):
+                    include_file = line[8:]
+                    include_path = file_path.parent / include_file
+                    domains.update(self._parse_v2fly_domain_file(include_path))
+                    continue
+
+                if line.startswith("domain:"):
+                    domain = line[7:]
+                elif line.startswith("full:"):
+                    domain = line[5:]
+                elif line.startswith("keyword:") or line.startswith("regexp:"):
+                    continue
+                else:
+                    domain = line
+
+                domain = domain.split("@")[0].strip()
+                if domain:
+                    domains.add(domain.lower())
+        return domains
+
+    def request(self, *args, **kwargs):
+        return request(*args, **kwargs)
+
+    def __str__(self):
+        return self.name
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(name='{self.name}')"