cloudcheck 8.5.1__cp313-cp313t-musllinux_1_2_aarch64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. cloudcheck/__init__.py +3 -0
  2. cloudcheck/cloudcheck.cpython-313t-aarch64-linux-musl.so +0 -0
  3. cloudcheck/helpers.py +249 -0
  4. cloudcheck/providers/__init__.py +51 -0
  5. cloudcheck/providers/akamai.py +33 -0
  6. cloudcheck/providers/alibaba.py +11 -0
  7. cloudcheck/providers/amazon.py +37 -0
  8. cloudcheck/providers/arvancloud.py +20 -0
  9. cloudcheck/providers/backblaze.py +11 -0
  10. cloudcheck/providers/base.py +268 -0
  11. cloudcheck/providers/cisco.py +39 -0
  12. cloudcheck/providers/cloudflare.py +40 -0
  13. cloudcheck/providers/cloudfront.py +18 -0
  14. cloudcheck/providers/dell.py +11 -0
  15. cloudcheck/providers/digitalocean.py +30 -0
  16. cloudcheck/providers/fastly.py +22 -0
  17. cloudcheck/providers/github.py +28 -0
  18. cloudcheck/providers/google.py +61 -0
  19. cloudcheck/providers/heroku.py +7 -0
  20. cloudcheck/providers/hetzner.py +18 -0
  21. cloudcheck/providers/hpe.py +12 -0
  22. cloudcheck/providers/huawei.py +17 -0
  23. cloudcheck/providers/ibm.py +57 -0
  24. cloudcheck/providers/imperva.py +24 -0
  25. cloudcheck/providers/kamatera.py +17 -0
  26. cloudcheck/providers/microsoft.py +37 -0
  27. cloudcheck/providers/oracle.py +31 -0
  28. cloudcheck/providers/ovh.py +15 -0
  29. cloudcheck/providers/rackspace.py +21 -0
  30. cloudcheck/providers/salesforce.py +15 -0
  31. cloudcheck/providers/scaleway.py +15 -0
  32. cloudcheck/providers/tencent.py +15 -0
  33. cloudcheck/providers/wasabi.py +15 -0
  34. cloudcheck/providers/zoho.py +25 -0
  35. cloudcheck-8.5.1.dist-info/METADATA +157 -0
  36. cloudcheck-8.5.1.dist-info/RECORD +38 -0
  37. cloudcheck-8.5.1.dist-info/WHEEL +4 -0
  38. cloudcheck.libs/libgcc_s-39080030.so.1 +0 -0
cloudcheck/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ from .cloudcheck import CloudCheck
2
+
3
+ __all__ = ["CloudCheck"]
cloudcheck/helpers.py ADDED
@@ -0,0 +1,249 @@
1
+ import ipaddress
2
+ import os
3
+ import requests
4
+ from pathlib import Path
5
+ from typing import List, Set, Union
6
+
7
+
8
+ def defrag_cidrs(
9
+ cidrs: List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]],
10
+ ) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
11
+ """
12
+ Defragment a list of CIDR blocks by merging adjacent networks.
13
+
14
+ Algorithm:
15
+ 1. Sort by network bits (prefix length)
16
+ 2. Iterate through pairs of adjacent networks
17
+ 3. If networks have equal network bits and can be merged into a larger network,
18
+ replace them with the merged network
19
+ 4. Repeat until no more merges are possible
20
+
21
+ Args:
22
+ cidrs: List of IPv4 or IPv6 network objects
23
+
24
+ Returns:
25
+ List of defragmented network objects
26
+ """
27
+ if not cidrs:
28
+ return []
29
+
30
+ # Convert to list and remove duplicates
31
+ networks = list(set(cidrs))
32
+
33
+ # Keep iterating until no more merges happen
34
+ changed = True
35
+ while changed:
36
+ changed = False
37
+
38
+ # Sort by network address
39
+ networks.sort(key=lambda x: (x.prefixlen, x.network_address.packed))
40
+
41
+ i = 0
42
+ while i < len(networks) - 1:
43
+ current = networks[i]
44
+ next_net = networks[i + 1]
45
+
46
+ # Check if we can merge these two networks
47
+ if _can_merge_networks(current, next_net):
48
+ # Create the merged network
49
+ merged = _merge_networks(current, next_net)
50
+
51
+ # Replace the two networks with the merged one
52
+ networks[i] = merged
53
+ networks.pop(i + 1)
54
+ changed = True
55
+ else:
56
+ i += 1
57
+
58
+ return networks
59
+
60
+
61
+ def _can_merge_networks(
62
+ net1: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
63
+ net2: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
64
+ ) -> bool:
65
+ """
66
+ Check if two networks can be merged into a larger network.
67
+
68
+ Two networks can be merged if:
69
+ 1. They have the same prefix length
70
+ 2. They are adjacent (one starts where the other ends)
71
+ 3. They can be combined into a network with prefix length - 1
72
+ """
73
+ # Must be same type (IPv4 or IPv6)
74
+ if net1.version != net2.version:
75
+ return False
76
+
77
+ # Must not be the same network
78
+ if net1 == net2:
79
+ return False
80
+
81
+ # Must have same prefix length
82
+ if net1.prefixlen != net2.prefixlen:
83
+ return False
84
+
85
+ # Must be adjacent networks
86
+ if not _are_adjacent_networks(net1, net2):
87
+ return False
88
+
89
+ return True
90
+
91
+
92
+ def _are_adjacent_networks(
93
+ net1: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
94
+ net2: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
95
+ ) -> bool:
96
+ """
97
+ Check if two networks are adjacent by creating two networks with sub-1 CIDR
98
+ and checking if they are equal.
99
+ """
100
+ # Must have same prefix length
101
+ if net1.prefixlen != net2.prefixlen:
102
+ return False
103
+
104
+ # Create two networks with sub-1 CIDR
105
+ new_prefixlen = net1.prefixlen - 1
106
+ if new_prefixlen < 0:
107
+ return False
108
+
109
+ # Create the two networks with the reduced prefix length using supernet
110
+ net1_parent = net1.supernet(prefixlen_diff=1)
111
+ net2_parent = net2.supernet(prefixlen_diff=1)
112
+
113
+ # If they are equal, the networks are adjacent
114
+ return net1_parent == net2_parent
115
+
116
+
117
+ def _merge_networks(
118
+ net1: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
119
+ net2: Union[ipaddress.IPv4Network, ipaddress.IPv6Network],
120
+ ) -> Union[ipaddress.IPv4Network, ipaddress.IPv6Network]:
121
+ """
122
+ Merge two adjacent networks into a larger network.
123
+ """
124
+ if net1 == net2:
125
+ raise ValueError("Networks must be different")
126
+
127
+ if not net1.version == net2.version:
128
+ raise ValueError("Networks must be the same version")
129
+
130
+ snet1 = net1.supernet(prefixlen_diff=1)
131
+ snet2 = net2.supernet(prefixlen_diff=1)
132
+ if not snet1 == snet2:
133
+ raise ValueError("Networks must be adjacent")
134
+
135
+ # Find the smaller network address
136
+ min_addr = min(net1.network_address, net2.network_address)
137
+
138
+ # Create the merged network with prefix length - 1
139
+ new_prefixlen = net1.prefixlen - 1
140
+ try:
141
+ return ipaddress.ip_network(f"{min_addr}/{new_prefixlen}")
142
+ except ValueError:
143
+ raise ValueError(
144
+ f"Failed to merge networks: {net1} (type: {type(net1)}) and {net2} (type: {type(net2)})"
145
+ )
146
+
147
+
148
+ def cidrs_to_strings(
149
+ cidrs: List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]],
150
+ ) -> List[str]:
151
+ """
152
+ Convert a list of network objects to string representations.
153
+
154
+ Args:
155
+ cidrs: List of network objects
156
+
157
+ Returns:
158
+ List of CIDR strings
159
+ """
160
+ return [str(cidr) for cidr in cidrs]
161
+
162
+
163
+ def strings_to_cidrs(
164
+ cidr_strings: List[str],
165
+ ) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
166
+ """
167
+ Convert a list of CIDR strings to network objects.
168
+
169
+ Args:
170
+ cidr_strings: List of CIDR strings
171
+
172
+ Returns:
173
+ List of network objects
174
+ """
175
+ networks = []
176
+ for cidr_str in cidr_strings:
177
+ try:
178
+ networks.append(ipaddress.ip_network(cidr_str, strict=False))
179
+ except ValueError:
180
+ # Skip invalid CIDR strings
181
+ continue
182
+ return networks
183
+
184
+
185
+ browser_base_headers = {
186
+ "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
187
+ "accept-language": "en-US,en;q=0.9",
188
+ "cache-control": "no-cache",
189
+ "pragma": "no-cache",
190
+ "priority": "u=0, i",
191
+ "referer": "https://www.google.com/",
192
+ "sec-ch-ua": '"Chromium";v="127", "Not)A;Brand";v="99"',
193
+ "sec-ch-ua-mobile": "?0",
194
+ "sec-ch-ua-platform": '"Linux"',
195
+ "sec-fetch-dest": "document",
196
+ "sec-fetch-mode": "navigate",
197
+ "sec-fetch-site": "cross-site",
198
+ "sec-fetch-user": "?1",
199
+ "upgrade-insecure-requests": "1",
200
+ "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
201
+ }
202
+
203
+
204
+ def request(url, include_api_key=False, browser_headers=False, **kwargs):
205
+ headers = kwargs.get("headers", {})
206
+ if browser_headers:
207
+ headers.update(browser_base_headers)
208
+ bbot_io_api_key = os.getenv("BBOT_IO_API_KEY")
209
+ if include_api_key and bbot_io_api_key:
210
+ headers["Authorization"] = f"Bearer {bbot_io_api_key}"
211
+ kwargs["headers"] = headers
212
+ return requests.get(url, **kwargs)
213
+
214
+
215
+ def parse_v2fly_domain_file(file_path: Path) -> Set[str]:
216
+ """Parse a domain list file and extract domains."""
217
+ print(f"Parsing {file_path}")
218
+ domains = set()
219
+ if not file_path.exists():
220
+ print(f"File {file_path} does not exist")
221
+ return domains
222
+
223
+ with open(file_path, "r", encoding="utf-8") as f:
224
+ for line in f:
225
+ line = line.strip()
226
+ # Handle inline comments by splitting on # and taking the first part
227
+ line = line.split("#")[0].strip()
228
+ if not line:
229
+ continue
230
+
231
+ if line.startswith("include:"):
232
+ include_file = line[8:]
233
+ include_path = file_path.parent / include_file
234
+ domains.update(parse_v2fly_domain_file(include_path))
235
+ continue
236
+
237
+ if line.startswith("domain:"):
238
+ domain = line[7:]
239
+ elif line.startswith("full:"):
240
+ domain = line[5:]
241
+ elif line.startswith("keyword:") or line.startswith("regexp:"):
242
+ continue
243
+ else:
244
+ domain = line
245
+
246
+ domain = domain.split("@")[0].strip()
247
+ if domain:
248
+ domains.add(domain.lower())
249
+ return domains
@@ -0,0 +1,51 @@
1
+ import importlib
2
+ from sys import stderr
3
+ from pathlib import Path
4
+ from typing import Dict, Type
5
+
6
+ from cloudcheck.providers.base import BaseProvider
7
+
8
+ # Dictionary to store loaded provider classes
9
+ _provider_classes: Dict[str, Type[BaseProvider]] = {}
10
+ _provider_instances: Dict[str, BaseProvider] = {}
11
+
12
+
13
+ def load_provider_classes() -> Dict[str, Type[BaseProvider]]:
14
+ """Dynamically load all cloud provider classes from the providers directory."""
15
+ global _provider_classes
16
+
17
+ if _provider_classes:
18
+ return _provider_classes
19
+
20
+ providers_path = Path(__file__).parent
21
+
22
+ for file in providers_path.glob("*.py"):
23
+ if file.stem in ("base", "__init__"):
24
+ continue
25
+
26
+ try:
27
+ import_path = f"cloudcheck.providers.{file.stem}"
28
+ module = importlib.import_module(import_path)
29
+
30
+ # Look for classes that inherit from BaseProvider
31
+ for attr_name in dir(module):
32
+ attr = getattr(module, attr_name)
33
+ if (
34
+ isinstance(attr, type)
35
+ and issubclass(attr, BaseProvider)
36
+ and attr != BaseProvider
37
+ ):
38
+ provider_name = attr.__name__
39
+ _provider_classes[provider_name] = attr
40
+
41
+ except Exception as e:
42
+ print(f"Failed to load provider from {file}: {e}", file=stderr)
43
+ raise
44
+
45
+ return _provider_classes
46
+
47
+
48
+ for provider_name, provider_class in load_provider_classes().items():
49
+ provider_instance = provider_class()
50
+ globals()[provider_name] = provider_instance
51
+ _provider_instances[provider_name] = provider_instance
@@ -0,0 +1,33 @@
1
+ import io
2
+ import zipfile
3
+ from cloudcheck.providers.base import BaseProvider
4
+ from typing import List
5
+
6
+
7
+ class Akamai(BaseProvider):
8
+ v2fly_company: str = "akamai"
9
+ tags: List[str] = ["cloud"]
10
+ # {"org_id": "AKAMAI-ARIN", "org_name": "Akamai Technologies, Inc.", "country": "US", "asns": [12222,16625,16702,17204,17334,18680,18717,20189,22207,22452,23454,23455,26008,30675,31984,32787,33047,35993,35994,36029,36183,393234,393560]}
11
+ # {"org_id": "ORG-AT1-RIPE", "org_name": "Akamai International B.V.", "country": "NL", "asns": [20940,21342,21357,21399,31107,31108,31109,31110,31377,33905,34164,34850,35204,39836,43639,48163,49249,49846,200005,213120]}
12
+ # {"org_id": "ORG-ATI1-AP-APNIC", "org_name": "Akamai Technologies, Inc.", "country": "US", "asns": [23903,24319,45757,55409,55770,63949,133103]}
13
+ org_ids: List[str] = [
14
+ "AKAMAI-ARIN",
15
+ "ORG-AT1-RIPE",
16
+ "ORG-ATI1-AP-APNIC",
17
+ ]
18
+
19
+ _ips_url = "https://techdocs.akamai.com/property-manager/pdfs/akamai_ipv4_ipv6_CIDRs-txt.zip"
20
+
21
+ def fetch_cidrs(self):
22
+ response = self.request(self._ips_url)
23
+ ranges = set()
24
+ content = getattr(response, "content", b"")
25
+ # Extract the contents of the zip file to memory
26
+ with zipfile.ZipFile(io.BytesIO(content)) as zip_file:
27
+ for filename in ("akamai_ipv4_CIDRs.txt", "akamai_ipv6_CIDRs.txt"):
28
+ with zip_file.open(filename) as f:
29
+ for line in f.read().splitlines():
30
+ line = line.decode(errors="ignore").strip()
31
+ if line:
32
+ ranges.add(line)
33
+ return list(ranges)
@@ -0,0 +1,11 @@
1
+ from cloudcheck.providers.base import BaseProvider
2
+ from typing import List
3
+
4
+
5
+ class Alibaba(BaseProvider):
6
+ v2fly_company: str = "alibaba"
7
+ tags: List[str] = ["cloud"]
8
+ # {"org_id": "ORG-ASEP1-AP-APNIC", "org_name": "Alibaba Cloud (Singapore) Private Limited", "country": "SG", "asns": [134963]}
9
+ org_ids: List[str] = [
10
+ "ORG-ASEP1-AP-APNIC",
11
+ ]
@@ -0,0 +1,37 @@
1
+ from cloudcheck.providers.base import BaseProvider
2
+ from typing import List, Dict
3
+
4
+
5
+ class Amazon(BaseProvider):
6
+ v2fly_company: str = "amazon"
7
+ org_ids: List[str] = [
8
+ "AMAZO-139-ARIN", # Amazon.com, Inc., US
9
+ "AMAZO-141-ARIN", # Amazon Technologies, Inc., US
10
+ "AMAZO-22-ARIN", # Amazon Web Services, Inc., US
11
+ "AMAZO-4-ARIN", # Amazon.com, Inc., US
12
+ "AMAZON-4-ARIN", # Amazon.com, Inc., US
13
+ "ARL-76-ARIN", # Amazon Robotics LLC, US
14
+ "ASL-830-ARIN", # Amazon.com Services, LLC, US
15
+ "AT-9049-ARIN", # Amazon Technologies Inc., US
16
+ "AT-9066-ARIN", # Amazon Technologies Inc., US
17
+ "ORG-AARP1-AP-APNIC", # Amazon Asia-Pacific Resources Private Limited, SG
18
+ "ORG-ACSP2-AP-APNIC", # Amazon Corporate Services Pty Ltd, AU
19
+ "ORG-ACTS1-AP-APNIC", # Amazon Connection Technology Services (Beijing) Co., LTD, CN
20
+ "ORG-ADSI1-RIPE", # Amazon Data Services Ireland Ltd, IE
21
+ "ORG-ADSJ1-AP-APNIC", # Amazon Data Services Japan KK, JP
22
+ "ORG-AI2-AP-APNIC", # Amazon.com, Inc., US
23
+ ]
24
+ tags: List[str] = ["cloud"]
25
+ _bucket_name_regex = r"[a-z0-9_][a-z0-9-\.]{1,61}[a-z0-9]"
26
+ regexes: Dict[str, List[str]] = {
27
+ "STORAGE_BUCKET_NAME": [_bucket_name_regex],
28
+ "STORAGE_BUCKET_HOSTNAME": [
29
+ r"(" + _bucket_name_regex + r")\.(s3-?(?:[a-z0-9-]*\.){1,2}amazonaws\.com)"
30
+ ],
31
+ }
32
+
33
+ _ips_url = "https://ip-ranges.amazonaws.com/ip-ranges.json"
34
+
35
+ def fetch_cidrs(self):
36
+ response = self.request(self._ips_url)
37
+ return list(set(p["ip_prefix"] for p in response.json()["prefixes"]))
@@ -0,0 +1,20 @@
1
+ from cloudcheck.providers.base import BaseProvider
2
+ from typing import List
3
+
4
+
5
+ class Arvancloud(BaseProvider):
6
+ domains: List[str] = ["arvancloud.ir"]
7
+ tags: List[str] = ["cdn"]
8
+ # {"org_id": "ORG-AGTL2-RIPE", "org_name": "ARVANCLOUD GLOBAL TECHNOLOGIES L.L.C", "country": "AE", "asns": [57568,208006,210296]}
9
+ org_ids: List[str] = [
10
+ "ORG-AGTL2-RIPE",
11
+ ]
12
+
13
+ _ips_url = "https://www.arvancloud.ir/en/ips.txt"
14
+
15
+ def fetch_cidrs(self):
16
+ response = self.request(self._ips_url)
17
+ ranges = set()
18
+ if getattr(response, "status_code", 0) == 200:
19
+ ranges.update(response.text.splitlines())
20
+ return list(ranges)
@@ -0,0 +1,11 @@
1
+ from cloudcheck.providers.base import BaseProvider
2
+ from typing import List
3
+
4
+
5
+ class Backblaze(BaseProvider):
6
+ tags: List[str] = ["cloud"]
7
+ # {"org_id": "BACKB-7-ARIN", "org_name": "Backblaze Inc", "country": "US", "asns": [40401,396865]}
8
+ org_ids: List[str] = [
9
+ "BACKB-7-ARIN",
10
+ ]
11
+ domains: List[str] = ["backblaze.com", "backblazeb2.com"]
@@ -0,0 +1,268 @@
1
+ import ipaddress
2
+ import os
3
+ import traceback
4
+ import subprocess
5
+ import time
6
+ from pathlib import Path
7
+ from typing import Dict, List, Union
8
+ from pydantic import BaseModel, field_validator, computed_field
9
+
10
+ from ..helpers import defrag_cidrs, parse_v2fly_domain_file, request
11
+
12
+
13
+ v2fly_repo_pulled = False
14
+
15
+
16
+ class BaseProvider(BaseModel):
17
+ """
18
+ Base class for all cloud providers.
19
+
20
+ Each provider inherits from this class and overrides any of the default values.
21
+ They can also override the update_cidrs() method to fetch cidrs from a different source.
22
+ """
23
+
24
+ # these values are static and always loaded from the class definition
25
+ regexes: Dict[str, List[str]] = {}
26
+ tags: List[str] = [] # Tags for the provider (e.g. "cdn", "waf", etc.)
27
+ org_ids: List[str] = [] # ASN Organization IDs (e.g. GOGL-ARIN)
28
+ v2fly_company: str = "" # Company name for v2fly domain fetching
29
+
30
+ # these values are dynamic and set by the update() method
31
+ last_updated: float = time.time()
32
+
33
+ # these we allow static values but they are later merged with dynamic values
34
+ asns: List[int] = []
35
+ cidrs: List[str] = []
36
+ domains: List[str] = []
37
+
38
+ @field_validator("cidrs")
39
+ @classmethod
40
+ def validate_cidrs(cls, value):
41
+ ips = []
42
+ for v in value:
43
+ try:
44
+ ips.append(ipaddress.ip_network(v, strict=False))
45
+ except ValueError:
46
+ print(f"Invalid CIDR: from {cls.__name__}: {v}")
47
+ continue
48
+ ips = [str(ip) for ip in defrag_cidrs(ips)]
49
+ return sorted(ips)
50
+
51
+ @field_validator("domains")
52
+ @classmethod
53
+ def validate_domains(cls, value):
54
+ return sorted(list(set([d.lower().strip(".") for d in value])))
55
+
56
+ @computed_field(return_type=str)
57
+ @property
58
+ def name(self):
59
+ return self.__class__.__name__
60
+
61
+ def __init__(self, **data):
62
+ super().__init__(**data)
63
+ self._cidrs = []
64
+ self._cache_dir = Path.home() / ".cache" / "cloudcheck"
65
+ self._repo_url = "https://github.com/v2fly/domain-list-community.git"
66
+ self._asndb_url = os.getenv("ASNDB_URL", "https://asndb.api.bbot.io/v1")
67
+ self._bbot_io_api_key = os.getenv("BBOT_IO_API_KEY")
68
+
69
+ def update(self):
70
+ print(f"Updating {self.name}")
71
+ errors = []
72
+ errors.extend(self.update_domains())
73
+ errors.extend(self.update_cidrs())
74
+ return errors
75
+
76
+ def update_domains(self):
77
+ # update dynamic domains
78
+ errors = []
79
+ if self.v2fly_company:
80
+ domains, errors = self.fetch_v2fly_domains()
81
+ if domains:
82
+ self.domains = sorted(list(set(self.domains + domains)))
83
+ else:
84
+ errors.append(
85
+ f"No v2fly domains were found for {self.name} (company name: {self.v2fly_company})"
86
+ )
87
+ return errors
88
+
89
+ def update_cidrs(self):
90
+ cidrs = set()
91
+ errors = []
92
+
93
+ # query by direct ASNs
94
+ if self.asns:
95
+ _cidrs, _errors = self.fetch_asns()
96
+ print(f"Got {len(_cidrs)} ASN cidrs for {self.name}'s ASNs {self.asns}")
97
+ if not _cidrs:
98
+ errors.append(
99
+ f"No ASN cidrs were found for {self.name}'s ASNs {self.asns}"
100
+ )
101
+ errors.extend(_errors)
102
+ cidrs.update(_cidrs)
103
+
104
+ # query by org IDs
105
+ if self.org_ids:
106
+ _cidrs, _asns, _errors = self.fetch_org_ids()
107
+ _asns = _asns.copy()
108
+ _asns.update(self.asns)
109
+ self.asns = list(sorted(_asns))
110
+ print(
111
+ f"Got {len(_cidrs)} org id cidrs for {self.name}'s org ids {self.org_ids}"
112
+ )
113
+ if not _cidrs:
114
+ errors.append(
115
+ f"No cidrs were found for {self.name}'s org ids {self.org_ids}"
116
+ )
117
+ errors.extend(_errors)
118
+ cidrs.update(_cidrs)
119
+
120
+ # fetch any dynamically-updated lists of CIDRs
121
+ try:
122
+ dynamic_cidrs = self.fetch_cidrs()
123
+ print(f"Got {len(dynamic_cidrs)} dynamic cidrs for {self.name}")
124
+ cidrs.update(dynamic_cidrs)
125
+ except Exception as e:
126
+ errors.append(
127
+ f"Failed to fetch dynamic cidrs for {self.name}: {e}:\n{traceback.format_exc()}"
128
+ )
129
+
130
+ # finally, put in any manually-specified CIDRs
131
+ print(f"Adding {len(self.cidrs)} manually-specified cidrs for {self.name}")
132
+ if self.cidrs:
133
+ cidrs.update(self.cidrs)
134
+
135
+ print(f"Total {len(cidrs)} cidrs for {self.name}")
136
+
137
+ try:
138
+ self.cidrs = self.validate_cidrs(cidrs)
139
+ except Exception as e:
140
+ errors.append(
141
+ f"Error validating ASN cidrs for {self.name}: {e}:\n{traceback.format_exc()}"
142
+ )
143
+
144
+ self.last_updated = time.time()
145
+
146
+ return errors
147
+
148
+ def fetch_org_ids(
149
+ self,
150
+ ) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
151
+ """Takes org_ids and populates the .asns and .cidrs attributes."""
152
+ errors = []
153
+ cidrs = set()
154
+ print(f"Fetching {len(self.org_ids)} org ids for {self.name}")
155
+ asns = set()
156
+ for org_id in self.org_ids:
157
+ print(f"Fetching cidrs for {org_id} from asndb")
158
+ try:
159
+ url = f"{self._asndb_url}/org/{org_id}"
160
+ print(f"Fetching {url}")
161
+ res = self.request(url, include_api_key=True)
162
+ print(f"{url} -> {res}: {res.text}")
163
+ j = res.json()
164
+ except Exception as e:
165
+ errors.append(
166
+ f"Failed to fetch cidrs for {org_id} from asndb: {e}:\n{traceback.format_exc()}"
167
+ )
168
+ continue
169
+ _asns = j.get("asns", [])
170
+ for asn in _asns:
171
+ asns.add(asn)
172
+ asn_cidrs, _errors = self.fetch_asn(asn)
173
+ errors.extend(_errors)
174
+ cidrs.update(asn_cidrs)
175
+ return cidrs, asns, errors
176
+
177
+ def fetch_asns(self) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
178
+ """Fetch CIDRs for a given list of ASNs from ASNDB."""
179
+ cidrs = []
180
+ errors = []
181
+ print(f"Fetching {len(self.asns)} ASNs for {self.name}")
182
+ for asn in self.asns:
183
+ asn_cidrs, _errors = self.fetch_asn(asn)
184
+ errors.extend(_errors)
185
+ cidrs.update(asn_cidrs)
186
+ return cidrs, errors
187
+
188
+ def fetch_asn(
189
+ self, asn: int
190
+ ) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
191
+ """Fetch CIDRs for a given ASN from ASNDB."""
192
+ cidrs = []
193
+ errors = []
194
+ url = f"{self._asndb_url}/asn/{asn}"
195
+ print(f"Fetching {url}")
196
+ try:
197
+ res = self.request(url, include_api_key=True)
198
+ print(f"{url} -> {res.text}")
199
+ j = res.json()
200
+ cidrs = j.get("subnets", [])
201
+ except Exception as e:
202
+ errors.append(
203
+ f"Failed to fetch cidrs for {asn} from asndb: {e}:\n{traceback.format_exc()}"
204
+ )
205
+ print(f"Got {len(cidrs)} cidrs for {asn}")
206
+ return cidrs, errors
207
+
208
+ def fetch_v2fly_domains(self) -> List[str]:
209
+ """Fetch domains from the v2fly community repository."""
210
+ if not self.v2fly_company:
211
+ return [], []
212
+
213
+ errors = []
214
+ repo_path, _success = self._ensure_v2fly_repo_cached()
215
+ company_file = repo_path / "data" / self.v2fly_company
216
+ try:
217
+ domains = parse_v2fly_domain_file(company_file)
218
+ except Exception as e:
219
+ errors.append(
220
+ f"Failed to parse {self.v2fly_company} domains: {e}:\n{traceback.format_exc()}"
221
+ )
222
+ return sorted(list(domains)), errors
223
+
224
+ def fetch_cidrs(self) -> List[str]:
225
+ """Fetch CIDRs from a custom source."""
226
+ return []
227
+
228
+ def fetch_domains(self) -> List[str]:
229
+ """Fetch domains from a custom source."""
230
+ return []
231
+
232
+ def _ensure_v2fly_repo_cached(self) -> Path:
233
+ """Ensure the community repo is cloned and up-to-date."""
234
+ global v2fly_repo_pulled
235
+ errors = []
236
+ repo_dir = self._cache_dir / "domain-list-community"
237
+ if not repo_dir.exists():
238
+ self._cache_dir.mkdir(parents=True, exist_ok=True)
239
+ try:
240
+ subprocess.run(
241
+ ["git", "clone", "--depth", "1", self._repo_url, str(repo_dir)],
242
+ check=True,
243
+ capture_output=True,
244
+ )
245
+ v2fly_repo_pulled = True
246
+ except subprocess.CalledProcessError as e:
247
+ errors.append(
248
+ f"Failed to clone v2fly repo: {e}:\n{traceback.format_exc()}"
249
+ )
250
+ elif not v2fly_repo_pulled:
251
+ try:
252
+ subprocess.run(
253
+ ["git", "pull"], cwd=repo_dir, check=True, capture_output=True
254
+ )
255
+ except subprocess.CalledProcessError as e:
256
+ errors.append(
257
+ f"Failed to pull v2fly repo: {e}:\n{traceback.format_exc()}"
258
+ )
259
+ return repo_dir, errors
260
+
261
+ def request(self, *args, **kwargs):
262
+ return request(*args, **kwargs)
263
+
264
+ def __str__(self):
265
+ return self.name
266
+
267
+ def __repr__(self):
268
+ return f"{self.__class__.__name__}(name='{self.name}')"