agg-search 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
agg_search/__init__.py ADDED
File without changes
agg_search/eval.py ADDED
@@ -0,0 +1,165 @@
1
+ #!/usr/bin/env python3
2
+ """RRF parameter sweep harness for agg-search — record once, sweep free.
3
+
4
+ record: run the judgment-set queries through the live engines ONCE and store
5
+ every engine's raw rows. This is the only step that spends engine
6
+ quota (Firecrawl credits, DeepSeek balance, ...).
7
+ sweep: replay the fixture entirely offline: RRF over a k x weights grid,
8
+ then (unless --no-jina) the real Jina rerank of each config's top-15,
9
+ scored against the relevance judgments. Jina tokens are the only
10
+ remaining cost (~2.5k tokens per config-query pair).
11
+
12
+ Usage:
13
+ python3 rrf_eval.py record --queries queries.json --out fixture.json
14
+ [--engines exa,brave,serper,deepseek,firecrawl,anysearch,tavily,searxng]
15
+ [--per-engine 5]
16
+ python3 rrf_eval.py sweep --fixture fixture.json --judgments judgments.json
17
+ [--top 10] [--no-jina] [--workers 4] [--diversify]
18
+
19
+ queries.json: ["query one", "query two", ...]
20
+ judgments.json: {"query one": ["https://relevant-url", ...], ...}
21
+ Relevance labels are built by pooling every engine's results for the query
22
+ and judging title/snippet relevance (TREC-style pooling; labels are
23
+ model-made, not human gold).
24
+ """
25
+ import argparse
26
+ import concurrent.futures as cf
27
+ import importlib.util
28
+ import json
29
+ import math
30
+ import sys
31
+ import time
32
+ from pathlib import Path
33
+
34
+ HERE = Path(__file__).resolve().parent
35
+
36
+ try:
37
+ from . import search as m
38
+ except ImportError: # run as a plain script from the repository root
39
+ import importlib.util
40
+
41
+ spec = importlib.util.spec_from_file_location("agg_search", HERE / "search.py")
42
+ m = importlib.util.module_from_spec(spec)
43
+ spec.loader.exec_module(m)
44
+
45
+ K_VALUES = (0, 10, 30, 60, 100)
46
+ WEIGHT_VARIANTS = {
47
+ "current (searxng .5)": {"searxng": 0.5},
48
+ "flat 1.0": {"searxng": 1.0},
49
+ "searxng .3": {"searxng": 0.3},
50
+ "exa 1.5": {"exa": 1.5, "searxng": 0.5},
51
+ "deepseek 1.5": {"deepseek": 1.5, "searxng": 0.5},
52
+ }
53
+
54
+
55
+ def record(a):
56
+ queries = json.loads(Path(a.queries).read_text(encoding="utf-8"))
57
+ engines = [e for e in a.engines.split(",") if e in m.ENGINES]
58
+ out = {"created": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
59
+ "per_engine": a.per_engine, "engines": engines, "queries": {}}
60
+ for q in queries:
61
+ results, status = m.recall(q, a.per_engine, engines, None)
62
+ out["queries"][q] = {"engines": results, "status": status}
63
+ print("recorded: %s | %s" % (q, "; ".join(
64
+ "%s=%s" % kv for kv in status.items())), file=sys.stderr)
65
+ Path(a.out).write_text(json.dumps(out, ensure_ascii=False, indent=1),
66
+ encoding="utf-8")
67
+ print("fixture -> %s" % a.out)
68
+
69
+
70
+ def ndcg10(order, relevant, top):
71
+ """Binary-gain nDCG@top for one ranked list."""
72
+ dcg = sum(1.0 / math.log2(i + 2)
73
+ for i, u in enumerate(order[:top]) if u in relevant)
74
+ ideal = sum(1.0 / math.log2(i + 2) for i in range(min(len(relevant), top)))
75
+ return dcg / ideal if ideal else 0.0
76
+
77
+
78
+ def evaluate_config(fx, judgments, k, weights, top, use_jina, workers,
79
+ diversify=False):
80
+ """One (k, weights) config over all queries; returns aggregate metrics."""
81
+ def one(item):
82
+ q, blob = item
83
+ ranked = m.rrf_fuse(blob["engines"], 15, weights, k=k,
84
+ diversify=diversify)
85
+ order15 = [u for u, _, _ in ranked]
86
+ rel = set(judgments.get(q, []))
87
+ rec15 = len(rel & set(order15)) / len(rel) if rel else 0.0
88
+ dom15 = len({m.domain_of(u) for u in order15})
89
+ if use_jina:
90
+ rr, _err = m.jina_rerank(q, ranked, top)
91
+ final = [u for u, _, _ in (rr if rr is not None else ranked)][:top]
92
+ else:
93
+ final = order15[:top]
94
+ return {"q": q, "recall15": rec15, "ndcg10": ndcg10(final, rel, top),
95
+ "hit5": 1.0 if rel & set(final[:5]) else 0.0,
96
+ "dom15": dom15,
97
+ "dom10": len({m.domain_of(u) for u in final})}
98
+
99
+ with cf.ThreadPoolExecutor(max_workers=workers) as ex:
100
+ per_query = list(ex.map(one, fx["queries"].items()))
101
+ n = len(per_query) or 1
102
+ return {"recall15": sum(x["recall15"] for x in per_query) / n,
103
+ "ndcg10": sum(x["ndcg10"] for x in per_query) / n,
104
+ "hit5": sum(x["hit5"] for x in per_query) / n,
105
+ "dom15": sum(x["dom15"] for x in per_query) / n,
106
+ "dom10": sum(x["dom10"] for x in per_query) / n,
107
+ "per_query": per_query}
108
+
109
+
110
+ def sweep(a):
111
+ fx = json.loads(Path(a.fixture).read_text(encoding="utf-8"))
112
+ judgments = json.loads(Path(a.judgments).read_text(encoding="utf-8"))
113
+ rows = []
114
+ for k in K_VALUES:
115
+ for name, w in WEIGHT_VARIANTS.items():
116
+ r = evaluate_config(fx, judgments, k, w, a.top,
117
+ not a.no_jina, a.workers,
118
+ diversify=a.diversify)
119
+ rows.append((k, name, r))
120
+ print("k=%-4s %-22s recall@15=%.3f nDCG@%s=%.3f hit@5=%.2f dom15=%.1f dom10=%.1f"
121
+ % (k, name, r["recall15"], a.top, r["ndcg10"], r["hit5"],
122
+ r["dom15"], r["dom10"]),
123
+ file=sys.stderr)
124
+ rows.sort(key=lambda x: (-x[2]["ndcg10"], -x[2]["recall15"]))
125
+ print("\n# RRF sweep — %d queries, Jina=%s, diversify=%s, top=%d"
126
+ % (len(fx["queries"]), "off" if a.no_jina else "v3.5",
127
+ "on" if a.diversify else "off", a.top))
128
+ print("\n| rank | k | weights | recall@15 | nDCG@%d | hit@5 | dom15 | dom10 |" % a.top)
129
+ print("|---|---|---|---|---|---|---|---|")
130
+ for i, (k, name, r) in enumerate(rows, 1):
131
+ print("| %d | %d | %s | %.3f | **%.3f** | %.2f | %.1f | %.1f |"
132
+ % (i, k, name, r["recall15"], r["ndcg10"], r["hit5"],
133
+ r["dom15"], r["dom10"]))
134
+ base = [r for k, name, r in rows if k == 60 and name.startswith("current")]
135
+ if base:
136
+ print("\ncurrent config (k=60, searxng .5): nDCG@%d=%.3f, recall@15=%.3f"
137
+ % (a.top, base[0]["ndcg10"], base[0]["recall15"]))
138
+
139
+
140
+ def main():
141
+ ap = argparse.ArgumentParser(description="RRF record/sweep harness")
142
+ sub = ap.add_subparsers(dest="cmd", required=True)
143
+ r = sub.add_parser("record")
144
+ r.add_argument("--queries", required=True)
145
+ r.add_argument("--out", required=True)
146
+ r.add_argument("--engines",
147
+ default="exa,brave,serper,deepseek,firecrawl,anysearch,tavily,searxng")
148
+ r.add_argument("--per-engine", type=int, default=5)
149
+ s = sub.add_parser("sweep")
150
+ s.add_argument("--fixture", required=True)
151
+ s.add_argument("--judgments", required=True)
152
+ s.add_argument("--top", type=int, default=10)
153
+ s.add_argument("--no-jina", action="store_true")
154
+ s.add_argument("--workers", type=int, default=4)
155
+ s.add_argument("--diversify", action="store_true",
156
+ help="eTLD+1 interleave before top-15 cut (measures the layer)")
157
+ a = ap.parse_args()
158
+ if a.cmd == "record":
159
+ record(a)
160
+ else:
161
+ sweep(a)
162
+
163
+
164
+ if __name__ == "__main__":
165
+ main()
agg_search/fetch.py ADDED
@@ -0,0 +1,226 @@
1
+ #!/usr/bin/env python3
2
+ """Tiered fetch orchestrator — host side, stdlib only, zero dependencies.
3
+
4
+ Rung 0 (when DEEPSEEK_API_KEY is set): DeepSeek's official Fetch API
5
+ (POST /v1/tools/fetch) — server-side fetch returning clean Markdown, one
6
+ HTTP call, no container. Thin/failed results fall through to the container
7
+ ladder. `--no-deepseek` skips it; `--tier deepseek` runs it alone.
8
+
9
+ Rungs 1-3 run inside the crawl-agent container (see docs/fetching.md);
10
+ the host never loads scrapling or a browser. This script only assembles and
11
+ runs `crawl-env` and relays its JSON output.
12
+
13
+ Exit codes: 0 = >=1 URL ok (prints JSON lines); 1 = all URLs failed;
14
+ 2 = launcher/image missing; 3 = blocked on every URL (container reports blocked).
15
+ """
16
+ import argparse
17
+ import json
18
+ import os
19
+ import re
20
+ import shutil
21
+ import subprocess
22
+ import sys
23
+ import time
24
+ import urllib.error
25
+ import urllib.request
26
+ from pathlib import Path
27
+
28
+ PKG_DIR = str(Path(__file__).resolve().parent)
29
+ CRAWL_ENV_FALLBACK = os.path.expanduser("~/.local/bin/crawl-env")
30
+ ENV_FILES = (os.path.expanduser("~/.config/agent-env"),
31
+ os.path.expanduser("~/.config/ai-keys.env"))
32
+
33
+
34
+ def getenv(key, default=None):
35
+ v = os.environ.get(key)
36
+ if v:
37
+ return v
38
+ for path in ENV_FILES:
39
+ try:
40
+ with open(path, encoding="utf-8") as f:
41
+ for line in f:
42
+ m = re.match(r'\s*(?:export\s+)?' + re.escape(key) + r'=(.*)',
43
+ line)
44
+ if m:
45
+ return m.group(1).strip().strip('"\'')
46
+ except OSError:
47
+ pass
48
+ return default
49
+
50
+
51
+ def squash(s):
52
+ return re.sub(r"\s+", " ", s or "").strip()
53
+
54
+
55
+ def deepseek_fetch(url, a, key):
56
+ """One Fetch-API call (with one retry on 408/429/5xx); returns a
57
+ container-shaped result dict."""
58
+ d = {"url": url, "tier_used": "deepseek", "status": None, "title": "",
59
+ "text": "", "markdown": "", "blocked": False, "error": ""}
60
+ if not key:
61
+ d["error"] = "no DEEPSEEK_API_KEY (see ENV.md)"
62
+ return d
63
+ base = getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com").rstrip("/")
64
+ req = urllib.request.Request(
65
+ base + "/v1/tools/fetch", method="POST",
66
+ data=json.dumps({"url": url}).encode(),
67
+ headers={"Content-Type": "application/json",
68
+ "Authorization": "Bearer " + key})
69
+ try:
70
+ try:
71
+ r = urllib.request.urlopen(req, timeout=a.timeout)
72
+ except urllib.error.HTTPError as e:
73
+ if e.code not in (408, 429) and e.code < 500:
74
+ raise
75
+ time.sleep(1.2)
76
+ r = urllib.request.urlopen(req, timeout=a.timeout)
77
+ with r:
78
+ d["status"] = r.status
79
+ body = json.loads(r.read().decode())
80
+ except urllib.error.HTTPError as e:
81
+ # 403 = rejected by security policy, 404 = no usable content
82
+ # (unavailable URL / site policy / unsupported type): both are
83
+ # escalation signals for the ladder, not hard errors.
84
+ d["status"] = e.code
85
+ d["blocked"] = e.code in (403, 404)
86
+ try:
87
+ msg = json.loads(e.read().decode())["error"]["message"]
88
+ except Exception: # noqa: BLE001 — error body is best-effort
89
+ msg = ""
90
+ d["error"] = "HTTP %d%s" % (e.code, (": " + msg[:200]) if msg else "")
91
+ return d
92
+ except Exception as e: # noqa: BLE001 — per-URL error lands in JSON
93
+ d["error"] = "%s: %s" % (type(e).__name__, e)
94
+ return d
95
+ md = body.get("content") or ""
96
+ text = squash(md)
97
+ if len(text) < 300 and len(md) < 300: # thin/empty page -> climb the ladder
98
+ d["blocked"] = True
99
+ d["error"] = "thin content (%d chars)" % len(text)
100
+ return d
101
+ d["title"] = body.get("title") or ""
102
+ d["text"] = text[:a.text_limit]
103
+ if a.out == "md":
104
+ d["markdown"] = md[:a.text_limit]
105
+ return d
106
+
107
+
108
+ def find_launcher():
109
+ hit = shutil.which("crawl-env")
110
+ if hit:
111
+ return hit
112
+ if os.path.isfile(CRAWL_ENV_FALLBACK) and os.access(CRAWL_ENV_FALLBACK, os.X_OK):
113
+ return CRAWL_ENV_FALLBACK
114
+ return None
115
+
116
+
117
+ def main():
118
+ ap = argparse.ArgumentParser(description="tiered page fetch: DeepSeek Fetch API rung + container ladder")
119
+ ap.add_argument("--url", action="append", required=True)
120
+ ap.add_argument("--tier", default="auto",
121
+ choices=("auto", "deepseek", "static", "dynamic", "stealth"))
122
+ ap.add_argument("--no-deepseek", action="store_true",
123
+ help="skip the DeepSeek Fetch API rung in auto mode")
124
+ ap.add_argument("--timeout", type=int, default=45)
125
+ ap.add_argument("--out", default="json", choices=("json", "md"))
126
+ ap.add_argument("--text-limit", type=int, default=8000)
127
+ ap.add_argument("--proxy", default=None)
128
+ ap.add_argument("--persist", action="store_true")
129
+ ap.add_argument("--headful", action="store_true")
130
+ ap.add_argument("--host-net", action="store_true",
131
+ help="share host network so a loopback-only proxy is reachable")
132
+ a = ap.parse_args()
133
+
134
+ ok = blocked = False
135
+ leftover = list(a.url)
136
+ ds_key = getenv("DEEPSEEK_API_KEY")
137
+ if a.tier == "deepseek" or (a.tier == "auto" and ds_key and not a.no_deepseek):
138
+ leftover = []
139
+ for u in a.url:
140
+ d = deepseek_fetch(u, a, ds_key)
141
+ if d["blocked"] or d["error"]:
142
+ leftover.append(u)
143
+ blocked = blocked or d["blocked"]
144
+ if a.tier == "deepseek": # auto defers silently to the ladder
145
+ print(json.dumps(d, ensure_ascii=False))
146
+ continue
147
+ print(json.dumps(d, ensure_ascii=False))
148
+ ok = True
149
+ if a.tier == "deepseek":
150
+ return 0 if ok else (3 if blocked else 1)
151
+
152
+ if leftover:
153
+ launcher = find_launcher()
154
+ if launcher is None:
155
+ print("crawl-env 未安装:见 docs/fetching.md", file=sys.stderr)
156
+ return 0 if ok else 2
157
+
158
+ if not leftover:
159
+ return 0 if ok else (3 if blocked else 1)
160
+
161
+ inner = ["python3", PKG_DIR + "/fetch_container.py",
162
+ "--tier", "auto" if a.tier == "auto" else a.tier,
163
+ "--timeout", str(a.timeout),
164
+ "--out", a.out, "--text-limit", str(a.text_limit)]
165
+ if a.persist:
166
+ inner.append("--persist-container")
167
+ if a.headful:
168
+ inner.append("--headful-container")
169
+ # Only an explicit --proxy is forwarded: the shell's HTTPS_PROXY typically
170
+ # points at a host-loopback proxy the container cannot reach. crawl-env
171
+ # forwards CRAWL_PROXY itself (with loopback rewriting).
172
+ if a.proxy:
173
+ inner += ["--proxy", a.proxy]
174
+ for u in leftover:
175
+ inner += ["--url", u]
176
+
177
+ cmd = [launcher]
178
+ if a.persist:
179
+ cmd.append("--persist")
180
+ if a.headful:
181
+ cmd.append("--headful")
182
+ if a.host_net:
183
+ cmd.append("--host-net")
184
+ cmd += ["--ro-mount", PKG_DIR, "--"] + inner
185
+
186
+ try:
187
+ p = subprocess.run(cmd, capture_output=True, text=True, timeout=a.timeout * len(leftover) + 120)
188
+ except FileNotFoundError:
189
+ print("docker 不可用(crawl-env 依赖 docker)", file=sys.stderr)
190
+ return 0 if ok else 2
191
+ except subprocess.TimeoutExpired:
192
+ print("fetch 超时:容器未在预期时间内返回", file=sys.stderr)
193
+ return 0 if ok else 1
194
+
195
+ if p.returncode == 125 or "docker: command not found" in (p.stderr or ""):
196
+ print("docker 不可用:见 docs/fetching.md 故障排查", file=sys.stderr)
197
+ return 0 if ok else 2
198
+ if "no such image" in (p.stderr or "") or "Unable to find image" in (p.stderr or ""):
199
+ print("crawl-agent 镜像缺失:docker build -t crawl-agent:latest "
200
+ "~/workspace/crawl-agent", file=sys.stderr)
201
+ return 0 if ok else 2
202
+
203
+ for line in (p.stdout or "").splitlines():
204
+ line = line.strip()
205
+ if not line.startswith("{"):
206
+ continue
207
+ print(line)
208
+ try:
209
+ d = json.loads(line)
210
+ except json.JSONDecodeError:
211
+ continue
212
+ if d.get("error"):
213
+ continue
214
+ if d.get("blocked"):
215
+ blocked = True
216
+ else:
217
+ ok = True
218
+ if p.stderr:
219
+ print(p.stderr, file=sys.stderr, end="")
220
+ if ok:
221
+ return 0
222
+ return 3 if blocked else 1
223
+
224
+
225
+ if __name__ == "__main__":
226
+ sys.exit(main())
@@ -0,0 +1,150 @@
1
+ #!/usr/bin/env python3
2
+ """In-container fetch — runs ONLY inside crawl-agent (has scrapling + browsers).
3
+
4
+ Tier ladder on --tier auto (first hit wins):
5
+ static -> Fetcher (curl_cffi, TLS impersonation, no browser)
6
+ dynamic -> DynamicFetcher (playwright chromium, JS rendering)
7
+ stealth -> StealthyFetcher (patchright chromium, Cloudflare solving)
8
+
9
+ Prints one JSON line per URL on stdout (scrapling INFO logs go to stderr).
10
+ Exit 0 = >=1 ok; 1 = all failed; 3 = blocked on every URL (non-200, block
11
+ markers, or JS-shell thin content after the last tier).
12
+ """
13
+ import argparse
14
+ import json
15
+ import os
16
+ import re
17
+ import sys
18
+
19
+ BLOCK_MARKERS = (
20
+ "just a moment", "cf-challenge", "attention required",
21
+ "verify you are human", "enable javascript and cookies",
22
+ "access denied", "captcha", "人机验证", "请开启javascript",
23
+ )
24
+
25
+
26
+ def squash(s):
27
+ return re.sub(r"\s+", " ", s or "").strip()
28
+
29
+
30
+ def needs_escalation(status, html, text):
31
+ """True when the page did not return real content (so the ladder climbs).
32
+
33
+ Thin-text heuristic catches JS shells: a 5 KB+ HTML skeleton rendering
34
+ under 300 chars of text is a client-rendered page, not an article.
35
+ """
36
+ if status != 200:
37
+ return True
38
+ low = (html or "").lower()
39
+ if any(m in low for m in BLOCK_MARKERS):
40
+ return True
41
+ return len(text or "") < 300 and len(html or "") > 3000
42
+
43
+
44
+ def result(url, tier_used, resp=None, include_md=False, error="", text_limit=8000):
45
+ d = {"url": url, "tier_used": tier_used, "status": None,
46
+ "title": "", "text": "", "markdown": "", "blocked": False,
47
+ "error": error}
48
+ if resp is None:
49
+ return d
50
+ d["status"] = resp.status
51
+ try:
52
+ d["title"] = squash(resp.css("title::text").get(default=""))
53
+ except Exception: # noqa: BLE001 — best-effort extraction, never aborts
54
+ pass
55
+ try:
56
+ d["text"] = squash(resp.get_all_text())[:text_limit]
57
+ except Exception: # noqa: BLE001
58
+ pass
59
+ if include_md:
60
+ try:
61
+ d["markdown"] = resp.markdown(main_content_only=True)[:text_limit]
62
+ except Exception: # noqa: BLE001
63
+ pass
64
+ return d
65
+
66
+
67
+ def fetch_one(url, args):
68
+ from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher # noqa: PLC0415
69
+
70
+ tiers = {"static": ("static",), "dynamic": ("dynamic",),
71
+ "stealth": ("stealth",)}.get(args.tier,
72
+ ("static", "dynamic", "stealth"))
73
+ explicit = args.proxy or os.environ.get("CRAWL_PROXY")
74
+ wildcard = os.environ.get("NO_PROXY") == "*" or os.environ.get("no_proxy") == "*"
75
+ # explicit CRAWL_PROXY/--proxy wins; otherwise wildcard NO_PROXY means direct;
76
+ # else fall back to the inherited HTTPS_PROXY.
77
+ proxy = explicit or (None if wildcard else os.environ.get("HTTPS_PROXY"))
78
+ headless = not (args.headful or args.headful_container)
79
+ last = None
80
+ for tier in tiers:
81
+ try:
82
+ if tier == "static":
83
+ kw = {"stealthy_headers": True, "impersonate": "chrome",
84
+ "timeout": args.timeout}
85
+ if proxy:
86
+ kw["proxy"] = proxy
87
+ resp = Fetcher.get(url, **kw)
88
+ elif tier == "dynamic":
89
+ kw = {"headless": headless, "network_idle": True,
90
+ "timeout": args.timeout * 1000, "extra_flags": ["--no-sandbox"],
91
+ "user_data_dir": "/tmp/home/chromium", "load_dom": True}
92
+ if proxy:
93
+ kw["proxy"] = proxy
94
+ resp = DynamicFetcher.fetch(url, **kw)
95
+ else:
96
+ kw = {"headless": headless, "solve_cloudflare": True,
97
+ "network_idle": True, "timeout": args.timeout * 1000,
98
+ "extra_flags": ["--no-sandbox"],
99
+ "user_data_dir": "/tmp/home/chromium"}
100
+ if proxy:
101
+ kw["proxy"] = proxy
102
+ resp = StealthyFetcher.fetch(url, **kw)
103
+ except Exception as e: # noqa: BLE001 — per-URL error lands in JSON
104
+ last = result(url, tier, error="%s: %s" % (type(e).__name__, e),
105
+ text_limit=args.text_limit)
106
+ continue
107
+ body = resp.body
108
+ html = body.decode("utf-8", "replace") if isinstance(body, bytes) else str(body)
109
+ d = result(url, tier, resp, include_md=(args.out == "md"),
110
+ text_limit=args.text_limit)
111
+ if not needs_escalation(resp.status, html, d["text"]):
112
+ return d
113
+ d["blocked"] = True
114
+ last = d
115
+ return last or result(url, None, error="no tier attempted",
116
+ text_limit=args.text_limit)
117
+
118
+
119
+ def main():
120
+ ap = argparse.ArgumentParser()
121
+ ap.add_argument("--url", action="append", required=True)
122
+ ap.add_argument("--tier", default="auto",
123
+ choices=("auto", "static", "dynamic", "stealth"))
124
+ ap.add_argument("--timeout", type=int, default=45)
125
+ ap.add_argument("--out", default="json", choices=("json", "md"))
126
+ ap.add_argument("--text-limit", type=int, default=8000)
127
+ ap.add_argument("--proxy", default=None)
128
+ ap.add_argument("--headful", action="store_true")
129
+ ap.add_argument("--headful-container", action="store_true")
130
+ ap.add_argument("--persist-container", action="store_true")
131
+ args = ap.parse_args()
132
+
133
+ n_ok, n_blocked = 0, 0
134
+ for u in args.url:
135
+ d = fetch_one(u, args)
136
+ print(json.dumps(d, ensure_ascii=False))
137
+ sys.stdout.flush()
138
+ if d["error"]:
139
+ continue
140
+ if d["blocked"]:
141
+ n_blocked += 1
142
+ else:
143
+ n_ok += 1
144
+ if n_ok:
145
+ return 0
146
+ return 3 if n_blocked else 1
147
+
148
+
149
+ if __name__ == "__main__":
150
+ sys.exit(main())