veil-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- veil_cli-0.1.0/PKG-INFO +8 -0
- veil_cli-0.1.0/app/__init__.py +1 -0
- veil_cli-0.1.0/app/__main__.py +4 -0
- veil_cli-0.1.0/app/cli.py +121 -0
- veil_cli-0.1.0/app/country_table.py +527 -0
- veil_cli-0.1.0/app/netinfo.py +109 -0
- veil_cli-0.1.0/app/ooni.py +158 -0
- veil_cli-0.1.0/app/ui.py +267 -0
- veil_cli-0.1.0/pyproject.toml +20 -0
- veil_cli-0.1.0/setup.cfg +4 -0
- veil_cli-0.1.0/veil_cli.egg-info/PKG-INFO +8 -0
- veil_cli-0.1.0/veil_cli.egg-info/SOURCES.txt +14 -0
- veil_cli-0.1.0/veil_cli.egg-info/dependency_links.txt +1 -0
- veil_cli-0.1.0/veil_cli.egg-info/entry_points.txt +2 -0
- veil_cli-0.1.0/veil_cli.egg-info/requires.txt +2 -0
- veil_cli-0.1.0/veil_cli.egg-info/top_level.txt +1 -0
veil_cli-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
from app import ui
|
|
7
|
+
from app.country_table import CANONICAL_NAMES, COUNTRY_CODES
|
|
8
|
+
from app.netinfo import gather_site_details
|
|
9
|
+
from app.ooni import OoniError, aggregate_by_country, aggregate_for_country
|
|
10
|
+
|
|
11
|
+
def resolve_country(raw: str) -> tuple[str, str] | None:
|
|
12
|
+
raw = raw.strip()
|
|
13
|
+
if not raw:
|
|
14
|
+
return None
|
|
15
|
+
if len(raw) == 2 and raw.upper() in CANONICAL_NAMES:
|
|
16
|
+
code = raw.upper()
|
|
17
|
+
return code, CANONICAL_NAMES[code]
|
|
18
|
+
for name, code in COUNTRY_CODES.items():
|
|
19
|
+
if name.lower() == raw.lower():
|
|
20
|
+
return code, CANONICAL_NAMES.get(code, name)
|
|
21
|
+
for name, code in COUNTRY_CODES.items():
|
|
22
|
+
if raw.lower() in name.lower():
|
|
23
|
+
return code, CANONICAL_NAMES.get(code, name)
|
|
24
|
+
return None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def run_scan(domain: str, country_raw: str | None) -> None:
|
|
28
|
+
with ui.spinner(f"looking up {domain}..."):
|
|
29
|
+
details = gather_site_details(domain)
|
|
30
|
+
ui.render_site_details(details)
|
|
31
|
+
|
|
32
|
+
if details.resolve_error:
|
|
33
|
+
return
|
|
34
|
+
|
|
35
|
+
if country_raw:
|
|
36
|
+
resolved = resolve_country(country_raw)
|
|
37
|
+
if not resolved:
|
|
38
|
+
ui.error_line(
|
|
39
|
+
f"couldn't recognize country '{country_raw}' — try a 2-letter "
|
|
40
|
+
f"code like IR, or a full name like Iran."
|
|
41
|
+
)
|
|
42
|
+
return
|
|
43
|
+
code, name = resolved
|
|
44
|
+
try:
|
|
45
|
+
with ui.spinner(f"querying OONI for {domain} in {name} (can take up to a minute)..."):
|
|
46
|
+
verdict = aggregate_for_country(details.domain, code)
|
|
47
|
+
except OoniError as exc:
|
|
48
|
+
ui.error_line(str(exc))
|
|
49
|
+
return
|
|
50
|
+
ui.render_country_verdict(verdict, name)
|
|
51
|
+
else:
|
|
52
|
+
try:
|
|
53
|
+
with ui.spinner(f"querying OONI across all countries for {domain} (can take up to a minute)..."):
|
|
54
|
+
verdicts, days_used = aggregate_by_country(details.domain)
|
|
55
|
+
except OoniError as exc:
|
|
56
|
+
ui.error_line(str(exc))
|
|
57
|
+
return
|
|
58
|
+
ui.render_country_list(verdicts, details.domain, days_used)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def interactive_loop() -> None:
|
|
62
|
+
ui.clear_screen()
|
|
63
|
+
ui.print_banner()
|
|
64
|
+
ui.print_intro_box()
|
|
65
|
+
while True:
|
|
66
|
+
try:
|
|
67
|
+
site = ui.prompt_site()
|
|
68
|
+
except (KeyboardInterrupt, EOFError):
|
|
69
|
+
ui.console.print()
|
|
70
|
+
break
|
|
71
|
+
|
|
72
|
+
if not site:
|
|
73
|
+
continue
|
|
74
|
+
if site.lower() in ("exit", "quit", "q"):
|
|
75
|
+
break
|
|
76
|
+
if site.lower() in ("help", "?"):
|
|
77
|
+
ui.print_help()
|
|
78
|
+
continue
|
|
79
|
+
|
|
80
|
+
try:
|
|
81
|
+
country = ui.prompt_country()
|
|
82
|
+
except (KeyboardInterrupt, EOFError):
|
|
83
|
+
ui.console.print()
|
|
84
|
+
break
|
|
85
|
+
|
|
86
|
+
ui.console.print()
|
|
87
|
+
run_scan(site, country or None)
|
|
88
|
+
ui.console.print()
|
|
89
|
+
|
|
90
|
+
ui.status_line("goodbye.")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
94
|
+
parser = argparse.ArgumentParser(
|
|
95
|
+
prog="veil",
|
|
96
|
+
description="Check whether a site is blocked in a given country, using OONI data.",
|
|
97
|
+
)
|
|
98
|
+
parser.add_argument("site", nargs="?", help="domain to check, e.g. youtube.com")
|
|
99
|
+
parser.add_argument("--country", "-c", help="country code or name, e.g. IR or Iran")
|
|
100
|
+
parser.add_argument(
|
|
101
|
+
"--list", "-l", action="store_true",
|
|
102
|
+
help="list all countries with blocking signal for this site (ignores --country)",
|
|
103
|
+
)
|
|
104
|
+
return parser
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def main() -> None:
|
|
108
|
+
parser = build_parser()
|
|
109
|
+
args = parser.parse_args()
|
|
110
|
+
|
|
111
|
+
if not args.site:
|
|
112
|
+
interactive_loop()
|
|
113
|
+
return
|
|
114
|
+
|
|
115
|
+
ui.print_banner()
|
|
116
|
+
country = None if args.list else args.country
|
|
117
|
+
run_scan(args.site, country)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
if __name__ == "__main__":
|
|
121
|
+
main()
|
|
@@ -0,0 +1,527 @@
|
|
|
1
|
+
COUNTRY_CODES = {
|
|
2
|
+
"Afghanistan": "AF",
|
|
3
|
+
"Albania": "AL",
|
|
4
|
+
"Algeria": "DZ",
|
|
5
|
+
"America": "US",
|
|
6
|
+
"American Samoa": "AS",
|
|
7
|
+
"Andorra": "AD",
|
|
8
|
+
"Angola": "AO",
|
|
9
|
+
"Anguilla": "AI",
|
|
10
|
+
"Antarctica": "AQ",
|
|
11
|
+
"Antigua and Barbuda": "AG",
|
|
12
|
+
"Argentina": "AR",
|
|
13
|
+
"Armenia": "AM",
|
|
14
|
+
"Aruba": "AW",
|
|
15
|
+
"Australia": "AU",
|
|
16
|
+
"Austria": "AT",
|
|
17
|
+
"Azerbaijan": "AZ",
|
|
18
|
+
"Bahamas": "BS",
|
|
19
|
+
"Bahrain": "BH",
|
|
20
|
+
"Bangladesh": "BD",
|
|
21
|
+
"Barbados": "BB",
|
|
22
|
+
"Belarus": "BY",
|
|
23
|
+
"Belgium": "BE",
|
|
24
|
+
"Belize": "BZ",
|
|
25
|
+
"Benin": "BJ",
|
|
26
|
+
"Bermuda": "BM",
|
|
27
|
+
"Bhutan": "BT",
|
|
28
|
+
"Bolivia": "BO",
|
|
29
|
+
"Bolivia, Plurinational State of": "BO",
|
|
30
|
+
"Bonaire, Sint Eustatius and Saba": "BQ",
|
|
31
|
+
"Bosnia and Herzegovina": "BA",
|
|
32
|
+
"Botswana": "BW",
|
|
33
|
+
"Bouvet Island": "BV",
|
|
34
|
+
"Brazil": "BR",
|
|
35
|
+
"Britain": "GB",
|
|
36
|
+
"British Indian Ocean Territory": "IO",
|
|
37
|
+
"Brunei": "BN",
|
|
38
|
+
"Brunei Darussalam": "BN",
|
|
39
|
+
"Bulgaria": "BG",
|
|
40
|
+
"Burkina Faso": "BF",
|
|
41
|
+
"Burundi": "BI",
|
|
42
|
+
"Cabo Verde": "CV",
|
|
43
|
+
"Cambodia": "KH",
|
|
44
|
+
"Cameroon": "CM",
|
|
45
|
+
"Canada": "CA",
|
|
46
|
+
"Cayman Islands": "KY",
|
|
47
|
+
"Central African Republic": "CF",
|
|
48
|
+
"Chad": "TD",
|
|
49
|
+
"Chile": "CL",
|
|
50
|
+
"China": "CN",
|
|
51
|
+
"Christmas Island": "CX",
|
|
52
|
+
"Cocos (Keeling) Islands": "CC",
|
|
53
|
+
"Colombia": "CO",
|
|
54
|
+
"Comoros": "KM",
|
|
55
|
+
"Congo": "CG",
|
|
56
|
+
"Congo, The Democratic Republic of the": "CD",
|
|
57
|
+
"Cook Islands": "CK",
|
|
58
|
+
"Costa Rica": "CR",
|
|
59
|
+
"Croatia": "HR",
|
|
60
|
+
"Cuba": "CU",
|
|
61
|
+
"Curaçao": "CW",
|
|
62
|
+
"Cyprus": "CY",
|
|
63
|
+
"Czech Republic": "CZ",
|
|
64
|
+
"Czechia": "CZ",
|
|
65
|
+
"Côte d'Ivoire": "CI",
|
|
66
|
+
"Denmark": "DK",
|
|
67
|
+
"Djibouti": "DJ",
|
|
68
|
+
"Dominica": "DM",
|
|
69
|
+
"Dominican Republic": "DO",
|
|
70
|
+
"Ecuador": "EC",
|
|
71
|
+
"Egypt": "EG",
|
|
72
|
+
"El Salvador": "SV",
|
|
73
|
+
"England": "GB",
|
|
74
|
+
"Equatorial Guinea": "GQ",
|
|
75
|
+
"Eritrea": "ER",
|
|
76
|
+
"Estonia": "EE",
|
|
77
|
+
"Eswatini": "SZ",
|
|
78
|
+
"Ethiopia": "ET",
|
|
79
|
+
"Falkland Islands (Malvinas)": "FK",
|
|
80
|
+
"Faroe Islands": "FO",
|
|
81
|
+
"Fiji": "FJ",
|
|
82
|
+
"Finland": "FI",
|
|
83
|
+
"France": "FR",
|
|
84
|
+
"French Guiana": "GF",
|
|
85
|
+
"French Polynesia": "PF",
|
|
86
|
+
"French Southern Territories": "TF",
|
|
87
|
+
"Gabon": "GA",
|
|
88
|
+
"Gambia": "GM",
|
|
89
|
+
"Georgia": "GE",
|
|
90
|
+
"Germany": "DE",
|
|
91
|
+
"Ghana": "GH",
|
|
92
|
+
"Gibraltar": "GI",
|
|
93
|
+
"Greece": "GR",
|
|
94
|
+
"Greenland": "GL",
|
|
95
|
+
"Grenada": "GD",
|
|
96
|
+
"Guadeloupe": "GP",
|
|
97
|
+
"Guam": "GU",
|
|
98
|
+
"Guatemala": "GT",
|
|
99
|
+
"Guernsey": "GG",
|
|
100
|
+
"Guinea": "GN",
|
|
101
|
+
"Guinea-Bissau": "GW",
|
|
102
|
+
"Guyana": "GY",
|
|
103
|
+
"Haiti": "HT",
|
|
104
|
+
"Heard Island and McDonald Islands": "HM",
|
|
105
|
+
"Holy See (Vatican City State)": "VA",
|
|
106
|
+
"Honduras": "HN",
|
|
107
|
+
"Hong Kong": "HK",
|
|
108
|
+
"Hungary": "HU",
|
|
109
|
+
"Iceland": "IS",
|
|
110
|
+
"India": "IN",
|
|
111
|
+
"Indonesia": "ID",
|
|
112
|
+
"Iran": "IR",
|
|
113
|
+
"Iran, Islamic Republic of": "IR",
|
|
114
|
+
"Iraq": "IQ",
|
|
115
|
+
"Ireland": "IE",
|
|
116
|
+
"Isle of Man": "IM",
|
|
117
|
+
"Israel": "IL",
|
|
118
|
+
"Italy": "IT",
|
|
119
|
+
"Ivory Coast": "CI",
|
|
120
|
+
"Jamaica": "JM",
|
|
121
|
+
"Japan": "JP",
|
|
122
|
+
"Jersey": "JE",
|
|
123
|
+
"Jordan": "JO",
|
|
124
|
+
"Kazakhstan": "KZ",
|
|
125
|
+
"Kenya": "KE",
|
|
126
|
+
"Kiribati": "KI",
|
|
127
|
+
"Korea, Democratic People's Republic of": "KP",
|
|
128
|
+
"Korea, Republic of": "KR",
|
|
129
|
+
"Kuwait": "KW",
|
|
130
|
+
"Kyrgyzstan": "KG",
|
|
131
|
+
"Lao People's Democratic Republic": "LA",
|
|
132
|
+
"Laos": "LA",
|
|
133
|
+
"Latvia": "LV",
|
|
134
|
+
"Lebanon": "LB",
|
|
135
|
+
"Lesotho": "LS",
|
|
136
|
+
"Liberia": "LR",
|
|
137
|
+
"Libya": "LY",
|
|
138
|
+
"Liechtenstein": "LI",
|
|
139
|
+
"Lithuania": "LT",
|
|
140
|
+
"Luxembourg": "LU",
|
|
141
|
+
"Macao": "MO",
|
|
142
|
+
"Macedonia": "MK",
|
|
143
|
+
"Madagascar": "MG",
|
|
144
|
+
"Malawi": "MW",
|
|
145
|
+
"Malaysia": "MY",
|
|
146
|
+
"Maldives": "MV",
|
|
147
|
+
"Mali": "ML",
|
|
148
|
+
"Malta": "MT",
|
|
149
|
+
"Marshall Islands": "MH",
|
|
150
|
+
"Martinique": "MQ",
|
|
151
|
+
"Mauritania": "MR",
|
|
152
|
+
"Mauritius": "MU",
|
|
153
|
+
"Mayotte": "YT",
|
|
154
|
+
"Mexico": "MX",
|
|
155
|
+
"Micronesia, Federated States of": "FM",
|
|
156
|
+
"Moldova": "MD",
|
|
157
|
+
"Moldova, Republic of": "MD",
|
|
158
|
+
"Monaco": "MC",
|
|
159
|
+
"Mongolia": "MN",
|
|
160
|
+
"Montenegro": "ME",
|
|
161
|
+
"Montserrat": "MS",
|
|
162
|
+
"Morocco": "MA",
|
|
163
|
+
"Mozambique": "MZ",
|
|
164
|
+
"Myanmar": "MM",
|
|
165
|
+
"Namibia": "NA",
|
|
166
|
+
"Nauru": "NR",
|
|
167
|
+
"Nepal": "NP",
|
|
168
|
+
"Netherlands": "NL",
|
|
169
|
+
"New Caledonia": "NC",
|
|
170
|
+
"New Zealand": "NZ",
|
|
171
|
+
"Nicaragua": "NI",
|
|
172
|
+
"Niger": "NE",
|
|
173
|
+
"Nigeria": "NG",
|
|
174
|
+
"Niue": "NU",
|
|
175
|
+
"Norfolk Island": "NF",
|
|
176
|
+
"North Korea": "KP",
|
|
177
|
+
"North Macedonia": "MK",
|
|
178
|
+
"Northern Mariana Islands": "MP",
|
|
179
|
+
"Norway": "NO",
|
|
180
|
+
"Oman": "OM",
|
|
181
|
+
"Pakistan": "PK",
|
|
182
|
+
"Palau": "PW",
|
|
183
|
+
"Palestine": "PS",
|
|
184
|
+
"Palestine, State of": "PS",
|
|
185
|
+
"Panama": "PA",
|
|
186
|
+
"Papua New Guinea": "PG",
|
|
187
|
+
"Paraguay": "PY",
|
|
188
|
+
"Peru": "PE",
|
|
189
|
+
"Philippines": "PH",
|
|
190
|
+
"Pitcairn": "PN",
|
|
191
|
+
"Poland": "PL",
|
|
192
|
+
"Portugal": "PT",
|
|
193
|
+
"Puerto Rico": "PR",
|
|
194
|
+
"Qatar": "QA",
|
|
195
|
+
"Romania": "RO",
|
|
196
|
+
"Russia": "RU",
|
|
197
|
+
"Russian Federation": "RU",
|
|
198
|
+
"Rwanda": "RW",
|
|
199
|
+
"Réunion": "RE",
|
|
200
|
+
"Saint Barthélemy": "BL",
|
|
201
|
+
"Saint Helena, Ascension and Tristan da Cunha": "SH",
|
|
202
|
+
"Saint Kitts and Nevis": "KN",
|
|
203
|
+
"Saint Lucia": "LC",
|
|
204
|
+
"Saint Martin (French part)": "MF",
|
|
205
|
+
"Saint Pierre and Miquelon": "PM",
|
|
206
|
+
"Saint Vincent and the Grenadines": "VC",
|
|
207
|
+
"Samoa": "WS",
|
|
208
|
+
"San Marino": "SM",
|
|
209
|
+
"Sao Tome and Principe": "ST",
|
|
210
|
+
"Saudi Arabia": "SA",
|
|
211
|
+
"Senegal": "SN",
|
|
212
|
+
"Serbia": "RS",
|
|
213
|
+
"Seychelles": "SC",
|
|
214
|
+
"Sierra Leone": "SL",
|
|
215
|
+
"Singapore": "SG",
|
|
216
|
+
"Sint Maarten (Dutch part)": "SX",
|
|
217
|
+
"Slovakia": "SK",
|
|
218
|
+
"Slovenia": "SI",
|
|
219
|
+
"Solomon Islands": "SB",
|
|
220
|
+
"Somalia": "SO",
|
|
221
|
+
"South Africa": "ZA",
|
|
222
|
+
"South Georgia and the South Sandwich Islands": "GS",
|
|
223
|
+
"South Korea": "KR",
|
|
224
|
+
"South Sudan": "SS",
|
|
225
|
+
"Spain": "ES",
|
|
226
|
+
"Sri Lanka": "LK",
|
|
227
|
+
"Sudan": "SD",
|
|
228
|
+
"Suriname": "SR",
|
|
229
|
+
"Svalbard and Jan Mayen": "SJ",
|
|
230
|
+
"Sweden": "SE",
|
|
231
|
+
"Switzerland": "CH",
|
|
232
|
+
"Syria": "SY",
|
|
233
|
+
"Syrian Arab Republic": "SY",
|
|
234
|
+
"Taiwan": "TW",
|
|
235
|
+
"Taiwan, Province of China": "TW",
|
|
236
|
+
"Tajikistan": "TJ",
|
|
237
|
+
"Tanzania": "TZ",
|
|
238
|
+
"Tanzania, United Republic of": "TZ",
|
|
239
|
+
"Thailand": "TH",
|
|
240
|
+
"Timor-Leste": "TL",
|
|
241
|
+
"Togo": "TG",
|
|
242
|
+
"Tokelau": "TK",
|
|
243
|
+
"Tonga": "TO",
|
|
244
|
+
"Trinidad and Tobago": "TT",
|
|
245
|
+
"Tunisia": "TN",
|
|
246
|
+
"Turkey": "TR",
|
|
247
|
+
"Turkmenistan": "TM",
|
|
248
|
+
"Turks and Caicos Islands": "TC",
|
|
249
|
+
"Tuvalu": "TV",
|
|
250
|
+
"Türkiye": "TR",
|
|
251
|
+
"UAE": "AE",
|
|
252
|
+
"UK": "GB",
|
|
253
|
+
"USA": "US",
|
|
254
|
+
"Uganda": "UG",
|
|
255
|
+
"Ukraine": "UA",
|
|
256
|
+
"United Arab Emirates": "AE",
|
|
257
|
+
"United Kingdom": "GB",
|
|
258
|
+
"United States": "US",
|
|
259
|
+
"United States Minor Outlying Islands": "UM",
|
|
260
|
+
"Uruguay": "UY",
|
|
261
|
+
"Uzbekistan": "UZ",
|
|
262
|
+
"Vanuatu": "VU",
|
|
263
|
+
"Venezuela": "VE",
|
|
264
|
+
"Venezuela, Bolivarian Republic of": "VE",
|
|
265
|
+
"Viet Nam": "VN",
|
|
266
|
+
"Vietnam": "VN",
|
|
267
|
+
"Virgin Islands, British": "VG",
|
|
268
|
+
"Virgin Islands, U.S.": "VI",
|
|
269
|
+
"Wallis and Futuna": "WF",
|
|
270
|
+
"Western Sahara": "EH",
|
|
271
|
+
"Yemen": "YE",
|
|
272
|
+
"Zambia": "ZM",
|
|
273
|
+
"Zimbabwe": "ZW",
|
|
274
|
+
"Åland Islands": "AX",
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
CANONICAL_NAMES = {
|
|
278
|
+
"AD": "Andorra",
|
|
279
|
+
"AE": "United Arab Emirates",
|
|
280
|
+
"AF": "Afghanistan",
|
|
281
|
+
"AG": "Antigua and Barbuda",
|
|
282
|
+
"AI": "Anguilla",
|
|
283
|
+
"AL": "Albania",
|
|
284
|
+
"AM": "Armenia",
|
|
285
|
+
"AO": "Angola",
|
|
286
|
+
"AQ": "Antarctica",
|
|
287
|
+
"AR": "Argentina",
|
|
288
|
+
"AS": "American Samoa",
|
|
289
|
+
"AT": "Austria",
|
|
290
|
+
"AU": "Australia",
|
|
291
|
+
"AW": "Aruba",
|
|
292
|
+
"AX": "Åland Islands",
|
|
293
|
+
"AZ": "Azerbaijan",
|
|
294
|
+
"BA": "Bosnia and Herzegovina",
|
|
295
|
+
"BB": "Barbados",
|
|
296
|
+
"BD": "Bangladesh",
|
|
297
|
+
"BE": "Belgium",
|
|
298
|
+
"BF": "Burkina Faso",
|
|
299
|
+
"BG": "Bulgaria",
|
|
300
|
+
"BH": "Bahrain",
|
|
301
|
+
"BI": "Burundi",
|
|
302
|
+
"BJ": "Benin",
|
|
303
|
+
"BL": "Saint Barthélemy",
|
|
304
|
+
"BM": "Bermuda",
|
|
305
|
+
"BN": "Brunei Darussalam",
|
|
306
|
+
"BO": "Bolivia",
|
|
307
|
+
"BQ": "Bonaire, Sint Eustatius and Saba",
|
|
308
|
+
"BR": "Brazil",
|
|
309
|
+
"BS": "Bahamas",
|
|
310
|
+
"BT": "Bhutan",
|
|
311
|
+
"BV": "Bouvet Island",
|
|
312
|
+
"BW": "Botswana",
|
|
313
|
+
"BY": "Belarus",
|
|
314
|
+
"BZ": "Belize",
|
|
315
|
+
"CA": "Canada",
|
|
316
|
+
"CC": "Cocos (Keeling) Islands",
|
|
317
|
+
"CD": "Congo, The Democratic Republic of the",
|
|
318
|
+
"CF": "Central African Republic",
|
|
319
|
+
"CG": "Congo",
|
|
320
|
+
"CH": "Switzerland",
|
|
321
|
+
"CI": "Côte d'Ivoire",
|
|
322
|
+
"CK": "Cook Islands",
|
|
323
|
+
"CL": "Chile",
|
|
324
|
+
"CM": "Cameroon",
|
|
325
|
+
"CN": "China",
|
|
326
|
+
"CO": "Colombia",
|
|
327
|
+
"CR": "Costa Rica",
|
|
328
|
+
"CU": "Cuba",
|
|
329
|
+
"CV": "Cabo Verde",
|
|
330
|
+
"CW": "Curaçao",
|
|
331
|
+
"CX": "Christmas Island",
|
|
332
|
+
"CY": "Cyprus",
|
|
333
|
+
"CZ": "Czechia",
|
|
334
|
+
"DE": "Germany",
|
|
335
|
+
"DJ": "Djibouti",
|
|
336
|
+
"DK": "Denmark",
|
|
337
|
+
"DM": "Dominica",
|
|
338
|
+
"DO": "Dominican Republic",
|
|
339
|
+
"DZ": "Algeria",
|
|
340
|
+
"EC": "Ecuador",
|
|
341
|
+
"EE": "Estonia",
|
|
342
|
+
"EG": "Egypt",
|
|
343
|
+
"EH": "Western Sahara",
|
|
344
|
+
"ER": "Eritrea",
|
|
345
|
+
"ES": "Spain",
|
|
346
|
+
"ET": "Ethiopia",
|
|
347
|
+
"FI": "Finland",
|
|
348
|
+
"FJ": "Fiji",
|
|
349
|
+
"FK": "Falkland Islands (Malvinas)",
|
|
350
|
+
"FM": "Micronesia, Federated States of",
|
|
351
|
+
"FO": "Faroe Islands",
|
|
352
|
+
"FR": "France",
|
|
353
|
+
"GA": "Gabon",
|
|
354
|
+
"GB": "United Kingdom",
|
|
355
|
+
"GD": "Grenada",
|
|
356
|
+
"GE": "Georgia",
|
|
357
|
+
"GF": "French Guiana",
|
|
358
|
+
"GG": "Guernsey",
|
|
359
|
+
"GH": "Ghana",
|
|
360
|
+
"GI": "Gibraltar",
|
|
361
|
+
"GL": "Greenland",
|
|
362
|
+
"GM": "Gambia",
|
|
363
|
+
"GN": "Guinea",
|
|
364
|
+
"GP": "Guadeloupe",
|
|
365
|
+
"GQ": "Equatorial Guinea",
|
|
366
|
+
"GR": "Greece",
|
|
367
|
+
"GS": "South Georgia and the South Sandwich Islands",
|
|
368
|
+
"GT": "Guatemala",
|
|
369
|
+
"GU": "Guam",
|
|
370
|
+
"GW": "Guinea-Bissau",
|
|
371
|
+
"GY": "Guyana",
|
|
372
|
+
"HK": "Hong Kong",
|
|
373
|
+
"HM": "Heard Island and McDonald Islands",
|
|
374
|
+
"HN": "Honduras",
|
|
375
|
+
"HR": "Croatia",
|
|
376
|
+
"HT": "Haiti",
|
|
377
|
+
"HU": "Hungary",
|
|
378
|
+
"ID": "Indonesia",
|
|
379
|
+
"IE": "Ireland",
|
|
380
|
+
"IL": "Israel",
|
|
381
|
+
"IM": "Isle of Man",
|
|
382
|
+
"IN": "India",
|
|
383
|
+
"IO": "British Indian Ocean Territory",
|
|
384
|
+
"IQ": "Iraq",
|
|
385
|
+
"IR": "Iran",
|
|
386
|
+
"IS": "Iceland",
|
|
387
|
+
"IT": "Italy",
|
|
388
|
+
"JE": "Jersey",
|
|
389
|
+
"JM": "Jamaica",
|
|
390
|
+
"JO": "Jordan",
|
|
391
|
+
"JP": "Japan",
|
|
392
|
+
"KE": "Kenya",
|
|
393
|
+
"KG": "Kyrgyzstan",
|
|
394
|
+
"KH": "Cambodia",
|
|
395
|
+
"KI": "Kiribati",
|
|
396
|
+
"KM": "Comoros",
|
|
397
|
+
"KN": "Saint Kitts and Nevis",
|
|
398
|
+
"KP": "North Korea",
|
|
399
|
+
"KR": "South Korea",
|
|
400
|
+
"KW": "Kuwait",
|
|
401
|
+
"KY": "Cayman Islands",
|
|
402
|
+
"KZ": "Kazakhstan",
|
|
403
|
+
"LA": "Laos",
|
|
404
|
+
"LB": "Lebanon",
|
|
405
|
+
"LC": "Saint Lucia",
|
|
406
|
+
"LI": "Liechtenstein",
|
|
407
|
+
"LK": "Sri Lanka",
|
|
408
|
+
"LR": "Liberia",
|
|
409
|
+
"LS": "Lesotho",
|
|
410
|
+
"LT": "Lithuania",
|
|
411
|
+
"LU": "Luxembourg",
|
|
412
|
+
"LV": "Latvia",
|
|
413
|
+
"LY": "Libya",
|
|
414
|
+
"MA": "Morocco",
|
|
415
|
+
"MC": "Monaco",
|
|
416
|
+
"MD": "Moldova",
|
|
417
|
+
"ME": "Montenegro",
|
|
418
|
+
"MF": "Saint Martin (French part)",
|
|
419
|
+
"MG": "Madagascar",
|
|
420
|
+
"MH": "Marshall Islands",
|
|
421
|
+
"MK": "North Macedonia",
|
|
422
|
+
"ML": "Mali",
|
|
423
|
+
"MM": "Myanmar",
|
|
424
|
+
"MN": "Mongolia",
|
|
425
|
+
"MO": "Macao",
|
|
426
|
+
"MP": "Northern Mariana Islands",
|
|
427
|
+
"MQ": "Martinique",
|
|
428
|
+
"MR": "Mauritania",
|
|
429
|
+
"MS": "Montserrat",
|
|
430
|
+
"MT": "Malta",
|
|
431
|
+
"MU": "Mauritius",
|
|
432
|
+
"MV": "Maldives",
|
|
433
|
+
"MW": "Malawi",
|
|
434
|
+
"MX": "Mexico",
|
|
435
|
+
"MY": "Malaysia",
|
|
436
|
+
"MZ": "Mozambique",
|
|
437
|
+
"NA": "Namibia",
|
|
438
|
+
"NC": "New Caledonia",
|
|
439
|
+
"NE": "Niger",
|
|
440
|
+
"NF": "Norfolk Island",
|
|
441
|
+
"NG": "Nigeria",
|
|
442
|
+
"NI": "Nicaragua",
|
|
443
|
+
"NL": "Netherlands",
|
|
444
|
+
"NO": "Norway",
|
|
445
|
+
"NP": "Nepal",
|
|
446
|
+
"NR": "Nauru",
|
|
447
|
+
"NU": "Niue",
|
|
448
|
+
"NZ": "New Zealand",
|
|
449
|
+
"OM": "Oman",
|
|
450
|
+
"PA": "Panama",
|
|
451
|
+
"PE": "Peru",
|
|
452
|
+
"PF": "French Polynesia",
|
|
453
|
+
"PG": "Papua New Guinea",
|
|
454
|
+
"PH": "Philippines",
|
|
455
|
+
"PK": "Pakistan",
|
|
456
|
+
"PL": "Poland",
|
|
457
|
+
"PM": "Saint Pierre and Miquelon",
|
|
458
|
+
"PN": "Pitcairn",
|
|
459
|
+
"PR": "Puerto Rico",
|
|
460
|
+
"PS": "Palestine, State of",
|
|
461
|
+
"PT": "Portugal",
|
|
462
|
+
"PW": "Palau",
|
|
463
|
+
"PY": "Paraguay",
|
|
464
|
+
"QA": "Qatar",
|
|
465
|
+
"RE": "Réunion",
|
|
466
|
+
"RO": "Romania",
|
|
467
|
+
"RS": "Serbia",
|
|
468
|
+
"RU": "Russian Federation",
|
|
469
|
+
"RW": "Rwanda",
|
|
470
|
+
"SA": "Saudi Arabia",
|
|
471
|
+
"SB": "Solomon Islands",
|
|
472
|
+
"SC": "Seychelles",
|
|
473
|
+
"SD": "Sudan",
|
|
474
|
+
"SE": "Sweden",
|
|
475
|
+
"SG": "Singapore",
|
|
476
|
+
"SH": "Saint Helena, Ascension and Tristan da Cunha",
|
|
477
|
+
"SI": "Slovenia",
|
|
478
|
+
"SJ": "Svalbard and Jan Mayen",
|
|
479
|
+
"SK": "Slovakia",
|
|
480
|
+
"SL": "Sierra Leone",
|
|
481
|
+
"SM": "San Marino",
|
|
482
|
+
"SN": "Senegal",
|
|
483
|
+
"SO": "Somalia",
|
|
484
|
+
"SR": "Suriname",
|
|
485
|
+
"SS": "South Sudan",
|
|
486
|
+
"ST": "Sao Tome and Principe",
|
|
487
|
+
"SV": "El Salvador",
|
|
488
|
+
"SX": "Sint Maarten (Dutch part)",
|
|
489
|
+
"SY": "Syria",
|
|
490
|
+
"SZ": "Eswatini",
|
|
491
|
+
"TC": "Turks and Caicos Islands",
|
|
492
|
+
"TD": "Chad",
|
|
493
|
+
"TF": "French Southern Territories",
|
|
494
|
+
"TG": "Togo",
|
|
495
|
+
"TH": "Thailand",
|
|
496
|
+
"TJ": "Tajikistan",
|
|
497
|
+
"TK": "Tokelau",
|
|
498
|
+
"TL": "Timor-Leste",
|
|
499
|
+
"TM": "Turkmenistan",
|
|
500
|
+
"TN": "Tunisia",
|
|
501
|
+
"TO": "Tonga",
|
|
502
|
+
"TR": "Türkiye",
|
|
503
|
+
"TT": "Trinidad and Tobago",
|
|
504
|
+
"TV": "Tuvalu",
|
|
505
|
+
"TW": "Taiwan",
|
|
506
|
+
"TZ": "Tanzania",
|
|
507
|
+
"UA": "Ukraine",
|
|
508
|
+
"UG": "Uganda",
|
|
509
|
+
"UM": "United States Minor Outlying Islands",
|
|
510
|
+
"US": "United States",
|
|
511
|
+
"UY": "Uruguay",
|
|
512
|
+
"UZ": "Uzbekistan",
|
|
513
|
+
"VA": "Holy See (Vatican City State)",
|
|
514
|
+
"VC": "Saint Vincent and the Grenadines",
|
|
515
|
+
"VE": "Venezuela",
|
|
516
|
+
"VG": "Virgin Islands, British",
|
|
517
|
+
"VI": "Virgin Islands, U.S.",
|
|
518
|
+
"VN": "Vietnam",
|
|
519
|
+
"VU": "Vanuatu",
|
|
520
|
+
"WF": "Wallis and Futuna",
|
|
521
|
+
"WS": "Samoa",
|
|
522
|
+
"YE": "Yemen",
|
|
523
|
+
"YT": "Mayotte",
|
|
524
|
+
"ZA": "South Africa",
|
|
525
|
+
"ZM": "Zambia",
|
|
526
|
+
"ZW": "Zimbabwe",
|
|
527
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import socket
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
|
|
9
|
+
HTTP_TIMEOUT = 8
|
|
10
|
+
GEOIP_URL = "http://ip-api.com/json/{ip}?fields=status,message,country,countryCode,isp,org,as"
|
|
11
|
+
TITLE_RE = re.compile(rb"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class SiteDetails:
|
|
16
|
+
domain: str
|
|
17
|
+
ip: str | None = None
|
|
18
|
+
resolve_error: str | None = None
|
|
19
|
+
http_status: int | None = None
|
|
20
|
+
http_error: str | None = None
|
|
21
|
+
server_header: str | None = None
|
|
22
|
+
title: str | None = None
|
|
23
|
+
final_url: str | None = None
|
|
24
|
+
isp: str | None = None
|
|
25
|
+
org: str | None = None
|
|
26
|
+
asn: str | None = None
|
|
27
|
+
host_country: str | None = None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _clean_domain(raw: str) -> str:
|
|
31
|
+
raw = raw.strip()
|
|
32
|
+
raw = re.sub(r"^\w+://", "", raw)
|
|
33
|
+
raw = raw.split("/")[0]
|
|
34
|
+
raw = raw.split(":")[0]
|
|
35
|
+
return raw
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def resolve(domain: str) -> tuple[str | None, str | None]:
|
|
39
|
+
try:
|
|
40
|
+
ip = socket.gethostbyname(domain)
|
|
41
|
+
return ip, None
|
|
42
|
+
except socket.gaierror as exc:
|
|
43
|
+
return None, str(exc)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def probe_http(domain: str) -> dict:
|
|
47
|
+
out = {"http_status": None, "http_error": None, "server_header": None,
|
|
48
|
+
"title": None, "final_url": None}
|
|
49
|
+
for scheme in ("https", "http"):
|
|
50
|
+
url = f"{scheme}://{domain}"
|
|
51
|
+
try:
|
|
52
|
+
resp = requests.get(
|
|
53
|
+
url,
|
|
54
|
+
timeout=HTTP_TIMEOUT,
|
|
55
|
+
headers={"User-Agent": "Mozilla/5.0 (compatible; veil-cli/0.1)"},
|
|
56
|
+
allow_redirects=True,
|
|
57
|
+
)
|
|
58
|
+
out["http_status"] = resp.status_code
|
|
59
|
+
out["server_header"] = resp.headers.get("Server")
|
|
60
|
+
out["final_url"] = resp.url
|
|
61
|
+
match = TITLE_RE.search(resp.content[:8192])
|
|
62
|
+
if match:
|
|
63
|
+
title = match.group(1).decode("utf-8", errors="ignore")
|
|
64
|
+
out["title"] = re.sub(r"\s+", " ", title).strip()[:120]
|
|
65
|
+
return out
|
|
66
|
+
except requests.RequestException as exc:
|
|
67
|
+
out["http_error"] = str(exc)
|
|
68
|
+
continue
|
|
69
|
+
return out
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def geoip_lookup(ip: str) -> dict:
|
|
73
|
+
out = {"isp": None, "org": None, "asn": None, "host_country": None}
|
|
74
|
+
try:
|
|
75
|
+
resp = requests.get(GEOIP_URL.format(ip=ip), timeout=6)
|
|
76
|
+
data = resp.json()
|
|
77
|
+
if data.get("status") == "success":
|
|
78
|
+
out["isp"] = data.get("isp")
|
|
79
|
+
out["org"] = data.get("org")
|
|
80
|
+
out["asn"] = data.get("as")
|
|
81
|
+
out["host_country"] = data.get("country")
|
|
82
|
+
except (requests.RequestException, ValueError):
|
|
83
|
+
pass
|
|
84
|
+
return out
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def gather_site_details(raw_domain: str) -> SiteDetails:
|
|
88
|
+
domain = _clean_domain(raw_domain)
|
|
89
|
+
details = SiteDetails(domain=domain)
|
|
90
|
+
|
|
91
|
+
ip, err = resolve(domain)
|
|
92
|
+
details.ip = ip
|
|
93
|
+
details.resolve_error = err
|
|
94
|
+
|
|
95
|
+
if ip:
|
|
96
|
+
http_info = probe_http(domain)
|
|
97
|
+
details.http_status = http_info["http_status"]
|
|
98
|
+
details.http_error = http_info["http_error"]
|
|
99
|
+
details.server_header = http_info["server_header"]
|
|
100
|
+
details.title = http_info["title"]
|
|
101
|
+
details.final_url = http_info["final_url"]
|
|
102
|
+
|
|
103
|
+
geo = geoip_lookup(ip)
|
|
104
|
+
details.isp = geo["isp"]
|
|
105
|
+
details.org = geo["org"]
|
|
106
|
+
details.asn = geo["asn"]
|
|
107
|
+
details.host_country = geo["host_country"]
|
|
108
|
+
|
|
109
|
+
return details
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import datetime as _dt
|
|
4
|
+
import time
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
|
|
9
|
+
API_BASE = "https://api.ooni.io/api/v1/aggregation"
|
|
10
|
+
USER_AGENT = "veil-cli/0.1 (+https://ooni.org)"
|
|
11
|
+
TIMEOUT = 120
|
|
12
|
+
|
|
13
|
+
MIN_RELIABLE_MEASUREMENTS = 5
|
|
14
|
+
ANOMALY_RATIO_THRESHOLD = 0.3
|
|
15
|
+
|
|
16
|
+
LIST_WINDOWS_DAYS = [365, 180, 90, 30, 14]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class CountryVerdict:
|
|
21
|
+
country_code: str
|
|
22
|
+
measurement_count: int
|
|
23
|
+
confirmed_count: int
|
|
24
|
+
anomaly_count: int
|
|
25
|
+
ok_count: int
|
|
26
|
+
failure_count: int
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def anomaly_ratio(self) -> float:
|
|
30
|
+
if self.measurement_count == 0:
|
|
31
|
+
return 0.0
|
|
32
|
+
return (self.anomaly_count + self.confirmed_count) / self.measurement_count
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def status(self) -> str:
|
|
36
|
+
if self.measurement_count < MIN_RELIABLE_MEASUREMENTS:
|
|
37
|
+
return "no_data"
|
|
38
|
+
if self.confirmed_count > 0:
|
|
39
|
+
return "blocked"
|
|
40
|
+
if self.anomaly_ratio >= ANOMALY_RATIO_THRESHOLD:
|
|
41
|
+
return "likely_blocked"
|
|
42
|
+
return "reachable"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class OoniError(RuntimeError):
|
|
46
|
+
pass
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _default_since_country(years_back: int = 1) -> str:
|
|
50
|
+
d = _dt.date.today() - _dt.timedelta(days=365 * years_back)
|
|
51
|
+
return d.isoformat()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _since_days_ago(days_back: int) -> str:
|
|
55
|
+
d = _dt.date.today() - _dt.timedelta(days=days_back)
|
|
56
|
+
return d.isoformat()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _default_until() -> str:
|
|
60
|
+
return _dt.date.today().isoformat()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _get(params: dict, retries: int = 2) -> dict:
|
|
64
|
+
last_exc = None
|
|
65
|
+
for attempt in range(retries):
|
|
66
|
+
try:
|
|
67
|
+
resp = requests.get(
|
|
68
|
+
API_BASE,
|
|
69
|
+
params=params,
|
|
70
|
+
headers={"User-Agent": USER_AGENT},
|
|
71
|
+
timeout=TIMEOUT,
|
|
72
|
+
)
|
|
73
|
+
if resp.status_code >= 500:
|
|
74
|
+
last_exc = OoniError(f"OONI's server is overloaded (HTTP {resp.status_code})")
|
|
75
|
+
time.sleep(2 * (attempt + 1))
|
|
76
|
+
continue
|
|
77
|
+
resp.raise_for_status()
|
|
78
|
+
return resp.json()
|
|
79
|
+
except requests.RequestException as exc:
|
|
80
|
+
last_exc = OoniError(f"couldn't reach the OONI API: {exc}")
|
|
81
|
+
time.sleep(2 * (attempt + 1))
|
|
82
|
+
except ValueError as exc:
|
|
83
|
+
raise OoniError("OONI API returned a non-JSON response") from exc
|
|
84
|
+
raise last_exc
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def aggregate_for_country(
|
|
88
|
+
domain: str,
|
|
89
|
+
country_code: str,
|
|
90
|
+
since: str | None = None,
|
|
91
|
+
until: str | None = None,
|
|
92
|
+
) -> CountryVerdict:
|
|
93
|
+
params = {
|
|
94
|
+
"domain": domain,
|
|
95
|
+
"probe_cc": country_code.upper(),
|
|
96
|
+
"test_name": "web_connectivity",
|
|
97
|
+
"since": since or _default_since_country(),
|
|
98
|
+
"until": until or _default_until(),
|
|
99
|
+
}
|
|
100
|
+
data = _get(params)
|
|
101
|
+
result = data.get("result") or {}
|
|
102
|
+
if isinstance(result, list):
|
|
103
|
+
result = result[0] if result else {}
|
|
104
|
+
return CountryVerdict(
|
|
105
|
+
country_code=country_code.upper(),
|
|
106
|
+
measurement_count=result.get("measurement_count", 0) or 0,
|
|
107
|
+
confirmed_count=result.get("confirmed_count", 0) or 0,
|
|
108
|
+
anomaly_count=result.get("anomaly_count", 0) or 0,
|
|
109
|
+
ok_count=result.get("ok_count", 0) or 0,
|
|
110
|
+
failure_count=result.get("failure_count", 0) or 0,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def aggregate_by_country(
|
|
115
|
+
domain: str,
|
|
116
|
+
until: str | None = None,
|
|
117
|
+
) -> tuple[list[CountryVerdict], int]:
|
|
118
|
+
"""Returns (verdicts, days_used) — tries the widest window first for the
|
|
119
|
+
best data coverage, and only falls back to a narrower window if OONI's
|
|
120
|
+
backend can't complete the query in time."""
|
|
121
|
+
last_exc = None
|
|
122
|
+
for days in LIST_WINDOWS_DAYS:
|
|
123
|
+
params = {
|
|
124
|
+
"domain": domain,
|
|
125
|
+
"axis_x": "probe_cc",
|
|
126
|
+
"test_name": "web_connectivity",
|
|
127
|
+
"since": _since_days_ago(days),
|
|
128
|
+
"until": until or _default_until(),
|
|
129
|
+
}
|
|
130
|
+
try:
|
|
131
|
+
data = _get(params, retries=2)
|
|
132
|
+
except OoniError as exc:
|
|
133
|
+
last_exc = exc
|
|
134
|
+
continue
|
|
135
|
+
|
|
136
|
+
rows = data.get("result") or []
|
|
137
|
+
verdicts = []
|
|
138
|
+
for row in rows:
|
|
139
|
+
cc = row.get("probe_cc")
|
|
140
|
+
if not cc or cc == "ZZ":
|
|
141
|
+
continue
|
|
142
|
+
verdicts.append(
|
|
143
|
+
CountryVerdict(
|
|
144
|
+
country_code=cc,
|
|
145
|
+
measurement_count=row.get("measurement_count", 0) or 0,
|
|
146
|
+
confirmed_count=row.get("confirmed_count", 0) or 0,
|
|
147
|
+
anomaly_count=row.get("anomaly_count", 0) or 0,
|
|
148
|
+
ok_count=row.get("ok_count", 0) or 0,
|
|
149
|
+
failure_count=row.get("failure_count", 0) or 0,
|
|
150
|
+
)
|
|
151
|
+
)
|
|
152
|
+
verdicts.sort(
|
|
153
|
+
key=lambda v: (v.confirmed_count, v.anomaly_count, v.measurement_count),
|
|
154
|
+
reverse=True,
|
|
155
|
+
)
|
|
156
|
+
return verdicts, days
|
|
157
|
+
|
|
158
|
+
raise last_exc
|
veil_cli-0.1.0/app/ui.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
from rich.align import Align
|
|
6
|
+
from rich.box import ROUNDED
|
|
7
|
+
from rich.console import Console, Group
|
|
8
|
+
from rich.padding import Padding
|
|
9
|
+
from rich.panel import Panel
|
|
10
|
+
from rich.prompt import Prompt
|
|
11
|
+
from rich.table import Table
|
|
12
|
+
from rich.text import Text
|
|
13
|
+
|
|
14
|
+
from app.ooni import CountryVerdict, MIN_RELIABLE_MEASUREMENTS
|
|
15
|
+
from app.netinfo import SiteDetails
|
|
16
|
+
|
|
17
|
+
WHITE = "white"
|
|
18
|
+
GREY = "grey62"
|
|
19
|
+
|
|
20
|
+
LOGO = r"""__ _____ ___ _
|
|
21
|
+
\ \ / / __|_ _| |
|
|
22
|
+
\ V /| _| | || |__
|
|
23
|
+
\_/ |___|___|____|
|
|
24
|
+
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
console = Console(highlight=False)
|
|
28
|
+
|
|
29
|
+
def clear_screen() -> None:
|
|
30
|
+
console.clear()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
STATUS_STYLE = {
|
|
34
|
+
"blocked": ("BLOCKED", "confirmed by OONI probes"),
|
|
35
|
+
"likely_blocked": ("LIKELY BLOCKED", "high anomaly rate in probe data"),
|
|
36
|
+
"reachable": ("REACHABLE", "no significant blocking signal"),
|
|
37
|
+
"no_data": ("NO DATA", "not enough OONI measurements to judge"),
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
STATUS_PRIORITY = {"blocked": 0, "likely_blocked": 1, "reachable": 2, "no_data": 3}
|
|
41
|
+
|
|
42
|
+
def print_banner() -> None:
|
|
43
|
+
logo = Text(LOGO, style=f"bold {WHITE}")
|
|
44
|
+
tagline = Text(" · an OSINT tool that checks whether a site is blocked in a given country · ", style=GREY)
|
|
45
|
+
console.print()
|
|
46
|
+
console.print(Align.center(logo))
|
|
47
|
+
console.print(Align.center(tagline))
|
|
48
|
+
console.print()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def print_intro_box() -> None:
|
|
52
|
+
body = Text()
|
|
53
|
+
body.append("Type a site to scan it, e.g. ", style=GREY)
|
|
54
|
+
body.append("youtube.com", style=f"bold {WHITE}")
|
|
55
|
+
body.append("\n")
|
|
56
|
+
body.append("Then a country code (or name), e.g. ", style=GREY)
|
|
57
|
+
body.append("PS", style=f"bold {WHITE}")
|
|
58
|
+
body.append(" or ", style=GREY)
|
|
59
|
+
body.append("Palestine", style=f"bold {WHITE}")
|
|
60
|
+
body.append(", leave blank to see all blocked countries.\n\n", style=GREY)
|
|
61
|
+
body.append("Commands: ", style=GREY)
|
|
62
|
+
body.append("help", style=f"bold {WHITE}")
|
|
63
|
+
body.append(" ", style=GREY)
|
|
64
|
+
body.append("exit", style=f"bold {WHITE}")
|
|
65
|
+
panel = Panel(
|
|
66
|
+
body,
|
|
67
|
+
title="[bold]what to do[/bold]",
|
|
68
|
+
title_align="left",
|
|
69
|
+
border_style=GREY,
|
|
70
|
+
box=ROUNDED,
|
|
71
|
+
padding=(1, 2),
|
|
72
|
+
)
|
|
73
|
+
console.print(panel)
|
|
74
|
+
console.print()
|
|
75
|
+
|
|
76
|
+
def prompt_in_box(label: str) -> str:
|
|
77
|
+
width = min(console.width - 4, 70)
|
|
78
|
+
inner_width = width - 2
|
|
79
|
+
top = f"╭{'─' * inner_width}╮"
|
|
80
|
+
bottom = f"╰{'─' * inner_width}╯"
|
|
81
|
+
|
|
82
|
+
prefix_visible = f" › {label} "
|
|
83
|
+
padding_len = max(0, inner_width - len(prefix_visible))
|
|
84
|
+
|
|
85
|
+
console.print(f"[{GREY}]{top}[/{GREY}]")
|
|
86
|
+
console.print(
|
|
87
|
+
f"[{GREY}]│[/{GREY}][bold {WHITE}] [/bold {WHITE}][{GREY}]› [/{GREY}]"
|
|
88
|
+
f"{label} {' ' * padding_len}[{GREY}]│[/{GREY}]"
|
|
89
|
+
)
|
|
90
|
+
console.print(f"[{GREY}]{bottom}[/{GREY}]")
|
|
91
|
+
|
|
92
|
+
input_col = len(prefix_visible) + 2
|
|
93
|
+
sys.stdout.write(f"\x1b[2A\x1b[{input_col}G")
|
|
94
|
+
sys.stdout.flush()
|
|
95
|
+
|
|
96
|
+
value = input().strip()
|
|
97
|
+
|
|
98
|
+
sys.stdout.write("\n")
|
|
99
|
+
sys.stdout.flush()
|
|
100
|
+
|
|
101
|
+
return value
|
|
102
|
+
|
|
103
|
+
def prompt_site() -> str:
|
|
104
|
+
return prompt_in_box("site to scan = ")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def prompt_country() -> str:
|
|
108
|
+
return prompt_in_box("country (leave it blank to list all) = ")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def status_line(msg: str, style: str = GREY) -> None:
|
|
112
|
+
console.print(f"[{style}]{msg}[/{style}]")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def error_line(msg: str) -> None:
|
|
116
|
+
console.print(f"[bold {WHITE}]✕ {msg}[/bold {WHITE}]")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def spinner(message: str):
|
|
120
|
+
return console.status(f"[{GREY}]{message}[/{GREY}]", spinner="dots")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def render_site_details(details: SiteDetails) -> None:
|
|
124
|
+
table = Table.grid(padding=(0, 1))
|
|
125
|
+
table.add_column(style=GREY, justify="right")
|
|
126
|
+
table.add_column(style=WHITE)
|
|
127
|
+
|
|
128
|
+
table.add_row("domain", details.domain)
|
|
129
|
+
if details.resolve_error:
|
|
130
|
+
table.add_row("resolve", f"[bold {WHITE}]failed — {details.resolve_error}[/bold {WHITE}]")
|
|
131
|
+
console.print(
|
|
132
|
+
Panel(table, title="[bold]site details[/bold]", title_align="left",
|
|
133
|
+
border_style=GREY, box=ROUNDED, padding=(1, 2))
|
|
134
|
+
)
|
|
135
|
+
return
|
|
136
|
+
|
|
137
|
+
table.add_row("ip", details.ip or "—")
|
|
138
|
+
if details.org or details.isp:
|
|
139
|
+
table.add_row("host", details.org or details.isp or "—")
|
|
140
|
+
if details.asn:
|
|
141
|
+
table.add_row("asn", details.asn)
|
|
142
|
+
if details.host_country:
|
|
143
|
+
table.add_row("hosted in", details.host_country)
|
|
144
|
+
if details.http_status:
|
|
145
|
+
table.add_row("http", f"[bold {WHITE}]{details.http_status}[/bold {WHITE}]"
|
|
146
|
+
f" ({details.final_url})")
|
|
147
|
+
elif details.http_error:
|
|
148
|
+
table.add_row("http", f"[{GREY}]unreachable from here[/{GREY}]")
|
|
149
|
+
if details.title:
|
|
150
|
+
table.add_row("title", details.title)
|
|
151
|
+
|
|
152
|
+
console.print(
|
|
153
|
+
Panel(table, title="[bold]site details[/bold]", title_align="left",
|
|
154
|
+
border_style=GREY, box=ROUNDED, padding=(1, 2))
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def render_country_verdict(verdict: CountryVerdict, country_label: str) -> None:
|
|
159
|
+
label, note = STATUS_STYLE[verdict.status]
|
|
160
|
+
header = Text()
|
|
161
|
+
header.append(f"{country_label} ({verdict.country_code}) ", style=f"bold {WHITE}")
|
|
162
|
+
header.append(label, style=f"bold {WHITE}")
|
|
163
|
+
|
|
164
|
+
table = Table.grid(padding=(0, 1))
|
|
165
|
+
table.add_column(style=GREY, justify="right")
|
|
166
|
+
table.add_column(style=WHITE)
|
|
167
|
+
table.add_row("measurements", str(verdict.measurement_count))
|
|
168
|
+
table.add_row("confirmed blocked", str(verdict.confirmed_count))
|
|
169
|
+
table.add_row("anomalous", str(verdict.anomaly_count))
|
|
170
|
+
table.add_row("ok", str(verdict.ok_count))
|
|
171
|
+
table.add_row("note", note)
|
|
172
|
+
|
|
173
|
+
console.print(
|
|
174
|
+
Panel(
|
|
175
|
+
Group(header, Text(""), table),
|
|
176
|
+
title="[bold]block status[/bold]",
|
|
177
|
+
title_align="left",
|
|
178
|
+
border_style=GREY,
|
|
179
|
+
box=ROUNDED,
|
|
180
|
+
padding=(1, 2),
|
|
181
|
+
)
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def render_country_list(
|
|
186
|
+
verdicts: list[CountryVerdict],
|
|
187
|
+
domain: str,
|
|
188
|
+
days_used: int | None = None,
|
|
189
|
+
page_size: int = 20,
|
|
190
|
+
) -> None:
|
|
191
|
+
if not verdicts:
|
|
192
|
+
console.print(Panel(
|
|
193
|
+
f"No OONI measurement data found for [bold {WHITE}]{domain}[/bold {WHITE}].",
|
|
194
|
+
border_style=GREY, box=ROUNDED, padding=(1, 2),
|
|
195
|
+
))
|
|
196
|
+
return
|
|
197
|
+
|
|
198
|
+
counts = {"blocked": 0, "likely_blocked": 0, "reachable": 0, "no_data": 0}
|
|
199
|
+
for v in verdicts:
|
|
200
|
+
counts[v.status] += 1
|
|
201
|
+
|
|
202
|
+
ordered = sorted(
|
|
203
|
+
verdicts,
|
|
204
|
+
key=lambda v: (STATUS_PRIORITY[v.status], -v.confirmed_count, -v.anomaly_count, -v.measurement_count),
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
pages = [ordered[i:i + page_size] for i in range(0, len(ordered), page_size)]
|
|
208
|
+
window_note = f" (last {days_used} days)" if days_used else ""
|
|
209
|
+
|
|
210
|
+
for page_num, page in enumerate(pages, start=1):
|
|
211
|
+
table = Table(
|
|
212
|
+
box=ROUNDED, border_style=GREY, show_lines=False,
|
|
213
|
+
title=f"countries with data for [bold {WHITE}]{domain}[/bold {WHITE}]{window_note}"
|
|
214
|
+
f" — page {page_num}/{len(pages)}",
|
|
215
|
+
title_justify="left",
|
|
216
|
+
)
|
|
217
|
+
table.add_column("country", style=WHITE)
|
|
218
|
+
table.add_column("status", justify="left", style=WHITE)
|
|
219
|
+
table.add_column("confirmed", justify="right", style=GREY)
|
|
220
|
+
table.add_column("anomalous", justify="right", style=GREY)
|
|
221
|
+
table.add_column("measurements", justify="right", style=GREY)
|
|
222
|
+
|
|
223
|
+
for v in page:
|
|
224
|
+
label, _ = STATUS_STYLE[v.status]
|
|
225
|
+
table.add_row(
|
|
226
|
+
v.country_code,
|
|
227
|
+
label,
|
|
228
|
+
str(v.confirmed_count),
|
|
229
|
+
str(v.anomaly_count),
|
|
230
|
+
str(v.measurement_count),
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
with console.capture() as capture:
|
|
234
|
+
console.print(table)
|
|
235
|
+
rendered = capture.get()
|
|
236
|
+
console.file.write(rendered)
|
|
237
|
+
console.file.flush()
|
|
238
|
+
table_lines = rendered.count("\n")
|
|
239
|
+
|
|
240
|
+
if page_num < len(pages):
|
|
241
|
+
console.input(f"[{GREY}]-- press Enter for next page ({page_num}/{len(pages)}) --[/{GREY}] ")
|
|
242
|
+
lines_to_clear = table_lines + 1
|
|
243
|
+
sys.stdout.write(f"\x1b[{lines_to_clear}A\x1b[0J")
|
|
244
|
+
sys.stdout.flush()
|
|
245
|
+
|
|
246
|
+
summary = Text()
|
|
247
|
+
summary.append(f"{counts['blocked']} blocked", style=f"bold {WHITE}")
|
|
248
|
+
summary.append(" · ", style=GREY)
|
|
249
|
+
summary.append(f"{counts['likely_blocked']} likely blocked", style=f"bold {WHITE}")
|
|
250
|
+
summary.append(" · ", style=GREY)
|
|
251
|
+
summary.append(f"{counts['reachable']} reachable", style=GREY)
|
|
252
|
+
summary.append(" · ", style=GREY)
|
|
253
|
+
summary.append(f"{counts['no_data']} no data (fewer than {MIN_RELIABLE_MEASUREMENTS} measurements)", style=GREY)
|
|
254
|
+
console.print(Padding(summary, (0, 1)))
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def print_help() -> None:
|
|
258
|
+
console.print(Panel(
|
|
259
|
+
f"[bold {WHITE}]veil <site>[/bold {WHITE}] [{GREY}][--country CC] [--list][/{GREY}]\n\n"
|
|
260
|
+
"Run with no arguments to start an interactive session.\n"
|
|
261
|
+
"In the session, just type a domain, then a country (or leave it "
|
|
262
|
+
"blank to see every country with blocking signal).\n\n"
|
|
263
|
+
f"[{GREY}]Data comes from OONI's crowdsourced probe network — treat "
|
|
264
|
+
f"results as a strong signal, not certainty.[/{GREY}]",
|
|
265
|
+
title="[bold]help[/bold]", title_align="left",
|
|
266
|
+
border_style=GREY, box=ROUNDED, padding=(1, 2),
|
|
267
|
+
))
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "veil-cli"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Check whether a site is blocked in a given country, using OONI data."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"requests>=2.28",
|
|
13
|
+
"rich>=13.0",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[project.scripts]
|
|
17
|
+
veil = "app.cli:main"
|
|
18
|
+
|
|
19
|
+
[tool.setuptools]
|
|
20
|
+
packages = ["app"]
|
veil_cli-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
pyproject.toml
|
|
2
|
+
app/__init__.py
|
|
3
|
+
app/__main__.py
|
|
4
|
+
app/cli.py
|
|
5
|
+
app/country_table.py
|
|
6
|
+
app/netinfo.py
|
|
7
|
+
app/ooni.py
|
|
8
|
+
app/ui.py
|
|
9
|
+
veil_cli.egg-info/PKG-INFO
|
|
10
|
+
veil_cli.egg-info/SOURCES.txt
|
|
11
|
+
veil_cli.egg-info/dependency_links.txt
|
|
12
|
+
veil_cli.egg-info/entry_points.txt
|
|
13
|
+
veil_cli.egg-info/requires.txt
|
|
14
|
+
veil_cli.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
app
|