agg-search 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agg_search/__init__.py +0 -0
- agg_search/eval.py +165 -0
- agg_search/fetch.py +226 -0
- agg_search/fetch_container.py +150 -0
- agg_search/mcp.py +171 -0
- agg_search/search.py +934 -0
- agg_search-0.1.0.dist-info/METADATA +147 -0
- agg_search-0.1.0.dist-info/RECORD +12 -0
- agg_search-0.1.0.dist-info/WHEEL +5 -0
- agg_search-0.1.0.dist-info/entry_points.txt +5 -0
- agg_search-0.1.0.dist-info/licenses/LICENSE +21 -0
- agg_search-0.1.0.dist-info/top_level.txt +1 -0
agg_search/__init__.py
ADDED
|
File without changes
|
agg_search/eval.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""RRF parameter sweep harness for agg-search — record once, sweep free.
|
|
3
|
+
|
|
4
|
+
record: run the judgment-set queries through the live engines ONCE and store
|
|
5
|
+
every engine's raw rows. This is the only step that spends engine
|
|
6
|
+
quota (Firecrawl credits, DeepSeek balance, ...).
|
|
7
|
+
sweep: replay the fixture entirely offline: RRF over a k x weights grid,
|
|
8
|
+
then (unless --no-jina) the real Jina rerank of each config's top-15,
|
|
9
|
+
scored against the relevance judgments. Jina tokens are the only
|
|
10
|
+
remaining cost (~2.5k tokens per config-query pair).
|
|
11
|
+
|
|
12
|
+
Usage:
|
|
13
|
+
python3 rrf_eval.py record --queries queries.json --out fixture.json
|
|
14
|
+
[--engines exa,brave,serper,deepseek,firecrawl,anysearch,tavily,searxng]
|
|
15
|
+
[--per-engine 5]
|
|
16
|
+
python3 rrf_eval.py sweep --fixture fixture.json --judgments judgments.json
|
|
17
|
+
[--top 10] [--no-jina] [--workers 4] [--diversify]
|
|
18
|
+
|
|
19
|
+
queries.json: ["query one", "query two", ...]
|
|
20
|
+
judgments.json: {"query one": ["https://relevant-url", ...], ...}
|
|
21
|
+
Relevance labels are built by pooling every engine's results for the query
|
|
22
|
+
and judging title/snippet relevance (TREC-style pooling; labels are
|
|
23
|
+
model-made, not human gold).
|
|
24
|
+
"""
|
|
25
|
+
import argparse
|
|
26
|
+
import concurrent.futures as cf
|
|
27
|
+
import importlib.util
|
|
28
|
+
import json
|
|
29
|
+
import math
|
|
30
|
+
import sys
|
|
31
|
+
import time
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
|
|
34
|
+
HERE = Path(__file__).resolve().parent
|
|
35
|
+
|
|
36
|
+
try:
|
|
37
|
+
from . import search as m
|
|
38
|
+
except ImportError: # run as a plain script from the repository root
|
|
39
|
+
import importlib.util
|
|
40
|
+
|
|
41
|
+
spec = importlib.util.spec_from_file_location("agg_search", HERE / "search.py")
|
|
42
|
+
m = importlib.util.module_from_spec(spec)
|
|
43
|
+
spec.loader.exec_module(m)
|
|
44
|
+
|
|
45
|
+
K_VALUES = (0, 10, 30, 60, 100)
|
|
46
|
+
WEIGHT_VARIANTS = {
|
|
47
|
+
"current (searxng .5)": {"searxng": 0.5},
|
|
48
|
+
"flat 1.0": {"searxng": 1.0},
|
|
49
|
+
"searxng .3": {"searxng": 0.3},
|
|
50
|
+
"exa 1.5": {"exa": 1.5, "searxng": 0.5},
|
|
51
|
+
"deepseek 1.5": {"deepseek": 1.5, "searxng": 0.5},
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def record(a):
|
|
56
|
+
queries = json.loads(Path(a.queries).read_text(encoding="utf-8"))
|
|
57
|
+
engines = [e for e in a.engines.split(",") if e in m.ENGINES]
|
|
58
|
+
out = {"created": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
|
59
|
+
"per_engine": a.per_engine, "engines": engines, "queries": {}}
|
|
60
|
+
for q in queries:
|
|
61
|
+
results, status = m.recall(q, a.per_engine, engines, None)
|
|
62
|
+
out["queries"][q] = {"engines": results, "status": status}
|
|
63
|
+
print("recorded: %s | %s" % (q, "; ".join(
|
|
64
|
+
"%s=%s" % kv for kv in status.items())), file=sys.stderr)
|
|
65
|
+
Path(a.out).write_text(json.dumps(out, ensure_ascii=False, indent=1),
|
|
66
|
+
encoding="utf-8")
|
|
67
|
+
print("fixture -> %s" % a.out)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def ndcg10(order, relevant, top):
|
|
71
|
+
"""Binary-gain nDCG@top for one ranked list."""
|
|
72
|
+
dcg = sum(1.0 / math.log2(i + 2)
|
|
73
|
+
for i, u in enumerate(order[:top]) if u in relevant)
|
|
74
|
+
ideal = sum(1.0 / math.log2(i + 2) for i in range(min(len(relevant), top)))
|
|
75
|
+
return dcg / ideal if ideal else 0.0
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def evaluate_config(fx, judgments, k, weights, top, use_jina, workers,
|
|
79
|
+
diversify=False):
|
|
80
|
+
"""One (k, weights) config over all queries; returns aggregate metrics."""
|
|
81
|
+
def one(item):
|
|
82
|
+
q, blob = item
|
|
83
|
+
ranked = m.rrf_fuse(blob["engines"], 15, weights, k=k,
|
|
84
|
+
diversify=diversify)
|
|
85
|
+
order15 = [u for u, _, _ in ranked]
|
|
86
|
+
rel = set(judgments.get(q, []))
|
|
87
|
+
rec15 = len(rel & set(order15)) / len(rel) if rel else 0.0
|
|
88
|
+
dom15 = len({m.domain_of(u) for u in order15})
|
|
89
|
+
if use_jina:
|
|
90
|
+
rr, _err = m.jina_rerank(q, ranked, top)
|
|
91
|
+
final = [u for u, _, _ in (rr if rr is not None else ranked)][:top]
|
|
92
|
+
else:
|
|
93
|
+
final = order15[:top]
|
|
94
|
+
return {"q": q, "recall15": rec15, "ndcg10": ndcg10(final, rel, top),
|
|
95
|
+
"hit5": 1.0 if rel & set(final[:5]) else 0.0,
|
|
96
|
+
"dom15": dom15,
|
|
97
|
+
"dom10": len({m.domain_of(u) for u in final})}
|
|
98
|
+
|
|
99
|
+
with cf.ThreadPoolExecutor(max_workers=workers) as ex:
|
|
100
|
+
per_query = list(ex.map(one, fx["queries"].items()))
|
|
101
|
+
n = len(per_query) or 1
|
|
102
|
+
return {"recall15": sum(x["recall15"] for x in per_query) / n,
|
|
103
|
+
"ndcg10": sum(x["ndcg10"] for x in per_query) / n,
|
|
104
|
+
"hit5": sum(x["hit5"] for x in per_query) / n,
|
|
105
|
+
"dom15": sum(x["dom15"] for x in per_query) / n,
|
|
106
|
+
"dom10": sum(x["dom10"] for x in per_query) / n,
|
|
107
|
+
"per_query": per_query}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def sweep(a):
|
|
111
|
+
fx = json.loads(Path(a.fixture).read_text(encoding="utf-8"))
|
|
112
|
+
judgments = json.loads(Path(a.judgments).read_text(encoding="utf-8"))
|
|
113
|
+
rows = []
|
|
114
|
+
for k in K_VALUES:
|
|
115
|
+
for name, w in WEIGHT_VARIANTS.items():
|
|
116
|
+
r = evaluate_config(fx, judgments, k, w, a.top,
|
|
117
|
+
not a.no_jina, a.workers,
|
|
118
|
+
diversify=a.diversify)
|
|
119
|
+
rows.append((k, name, r))
|
|
120
|
+
print("k=%-4s %-22s recall@15=%.3f nDCG@%s=%.3f hit@5=%.2f dom15=%.1f dom10=%.1f"
|
|
121
|
+
% (k, name, r["recall15"], a.top, r["ndcg10"], r["hit5"],
|
|
122
|
+
r["dom15"], r["dom10"]),
|
|
123
|
+
file=sys.stderr)
|
|
124
|
+
rows.sort(key=lambda x: (-x[2]["ndcg10"], -x[2]["recall15"]))
|
|
125
|
+
print("\n# RRF sweep — %d queries, Jina=%s, diversify=%s, top=%d"
|
|
126
|
+
% (len(fx["queries"]), "off" if a.no_jina else "v3.5",
|
|
127
|
+
"on" if a.diversify else "off", a.top))
|
|
128
|
+
print("\n| rank | k | weights | recall@15 | nDCG@%d | hit@5 | dom15 | dom10 |" % a.top)
|
|
129
|
+
print("|---|---|---|---|---|---|---|---|")
|
|
130
|
+
for i, (k, name, r) in enumerate(rows, 1):
|
|
131
|
+
print("| %d | %d | %s | %.3f | **%.3f** | %.2f | %.1f | %.1f |"
|
|
132
|
+
% (i, k, name, r["recall15"], r["ndcg10"], r["hit5"],
|
|
133
|
+
r["dom15"], r["dom10"]))
|
|
134
|
+
base = [r for k, name, r in rows if k == 60 and name.startswith("current")]
|
|
135
|
+
if base:
|
|
136
|
+
print("\ncurrent config (k=60, searxng .5): nDCG@%d=%.3f, recall@15=%.3f"
|
|
137
|
+
% (a.top, base[0]["ndcg10"], base[0]["recall15"]))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def main():
|
|
141
|
+
ap = argparse.ArgumentParser(description="RRF record/sweep harness")
|
|
142
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
143
|
+
r = sub.add_parser("record")
|
|
144
|
+
r.add_argument("--queries", required=True)
|
|
145
|
+
r.add_argument("--out", required=True)
|
|
146
|
+
r.add_argument("--engines",
|
|
147
|
+
default="exa,brave,serper,deepseek,firecrawl,anysearch,tavily,searxng")
|
|
148
|
+
r.add_argument("--per-engine", type=int, default=5)
|
|
149
|
+
s = sub.add_parser("sweep")
|
|
150
|
+
s.add_argument("--fixture", required=True)
|
|
151
|
+
s.add_argument("--judgments", required=True)
|
|
152
|
+
s.add_argument("--top", type=int, default=10)
|
|
153
|
+
s.add_argument("--no-jina", action="store_true")
|
|
154
|
+
s.add_argument("--workers", type=int, default=4)
|
|
155
|
+
s.add_argument("--diversify", action="store_true",
|
|
156
|
+
help="eTLD+1 interleave before top-15 cut (measures the layer)")
|
|
157
|
+
a = ap.parse_args()
|
|
158
|
+
if a.cmd == "record":
|
|
159
|
+
record(a)
|
|
160
|
+
else:
|
|
161
|
+
sweep(a)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
if __name__ == "__main__":
|
|
165
|
+
main()
|
agg_search/fetch.py
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Tiered fetch orchestrator — host side, stdlib only, zero dependencies.
|
|
3
|
+
|
|
4
|
+
Rung 0 (when DEEPSEEK_API_KEY is set): DeepSeek's official Fetch API
|
|
5
|
+
(POST /v1/tools/fetch) — server-side fetch returning clean Markdown, one
|
|
6
|
+
HTTP call, no container. Thin/failed results fall through to the container
|
|
7
|
+
ladder. `--no-deepseek` skips it; `--tier deepseek` runs it alone.
|
|
8
|
+
|
|
9
|
+
Rungs 1-3 run inside the crawl-agent container (see docs/fetching.md);
|
|
10
|
+
the host never loads scrapling or a browser. This script only assembles and
|
|
11
|
+
runs `crawl-env` and relays its JSON output.
|
|
12
|
+
|
|
13
|
+
Exit codes: 0 = >=1 URL ok (prints JSON lines); 1 = all URLs failed;
|
|
14
|
+
2 = launcher/image missing; 3 = blocked on every URL (container reports blocked).
|
|
15
|
+
"""
|
|
16
|
+
import argparse
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import shutil
|
|
21
|
+
import subprocess
|
|
22
|
+
import sys
|
|
23
|
+
import time
|
|
24
|
+
import urllib.error
|
|
25
|
+
import urllib.request
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
PKG_DIR = str(Path(__file__).resolve().parent)
|
|
29
|
+
CRAWL_ENV_FALLBACK = os.path.expanduser("~/.local/bin/crawl-env")
|
|
30
|
+
ENV_FILES = (os.path.expanduser("~/.config/agent-env"),
|
|
31
|
+
os.path.expanduser("~/.config/ai-keys.env"))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def getenv(key, default=None):
|
|
35
|
+
v = os.environ.get(key)
|
|
36
|
+
if v:
|
|
37
|
+
return v
|
|
38
|
+
for path in ENV_FILES:
|
|
39
|
+
try:
|
|
40
|
+
with open(path, encoding="utf-8") as f:
|
|
41
|
+
for line in f:
|
|
42
|
+
m = re.match(r'\s*(?:export\s+)?' + re.escape(key) + r'=(.*)',
|
|
43
|
+
line)
|
|
44
|
+
if m:
|
|
45
|
+
return m.group(1).strip().strip('"\'')
|
|
46
|
+
except OSError:
|
|
47
|
+
pass
|
|
48
|
+
return default
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def squash(s):
|
|
52
|
+
return re.sub(r"\s+", " ", s or "").strip()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def deepseek_fetch(url, a, key):
|
|
56
|
+
"""One Fetch-API call (with one retry on 408/429/5xx); returns a
|
|
57
|
+
container-shaped result dict."""
|
|
58
|
+
d = {"url": url, "tier_used": "deepseek", "status": None, "title": "",
|
|
59
|
+
"text": "", "markdown": "", "blocked": False, "error": ""}
|
|
60
|
+
if not key:
|
|
61
|
+
d["error"] = "no DEEPSEEK_API_KEY (see ENV.md)"
|
|
62
|
+
return d
|
|
63
|
+
base = getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com").rstrip("/")
|
|
64
|
+
req = urllib.request.Request(
|
|
65
|
+
base + "/v1/tools/fetch", method="POST",
|
|
66
|
+
data=json.dumps({"url": url}).encode(),
|
|
67
|
+
headers={"Content-Type": "application/json",
|
|
68
|
+
"Authorization": "Bearer " + key})
|
|
69
|
+
try:
|
|
70
|
+
try:
|
|
71
|
+
r = urllib.request.urlopen(req, timeout=a.timeout)
|
|
72
|
+
except urllib.error.HTTPError as e:
|
|
73
|
+
if e.code not in (408, 429) and e.code < 500:
|
|
74
|
+
raise
|
|
75
|
+
time.sleep(1.2)
|
|
76
|
+
r = urllib.request.urlopen(req, timeout=a.timeout)
|
|
77
|
+
with r:
|
|
78
|
+
d["status"] = r.status
|
|
79
|
+
body = json.loads(r.read().decode())
|
|
80
|
+
except urllib.error.HTTPError as e:
|
|
81
|
+
# 403 = rejected by security policy, 404 = no usable content
|
|
82
|
+
# (unavailable URL / site policy / unsupported type): both are
|
|
83
|
+
# escalation signals for the ladder, not hard errors.
|
|
84
|
+
d["status"] = e.code
|
|
85
|
+
d["blocked"] = e.code in (403, 404)
|
|
86
|
+
try:
|
|
87
|
+
msg = json.loads(e.read().decode())["error"]["message"]
|
|
88
|
+
except Exception: # noqa: BLE001 — error body is best-effort
|
|
89
|
+
msg = ""
|
|
90
|
+
d["error"] = "HTTP %d%s" % (e.code, (": " + msg[:200]) if msg else "")
|
|
91
|
+
return d
|
|
92
|
+
except Exception as e: # noqa: BLE001 — per-URL error lands in JSON
|
|
93
|
+
d["error"] = "%s: %s" % (type(e).__name__, e)
|
|
94
|
+
return d
|
|
95
|
+
md = body.get("content") or ""
|
|
96
|
+
text = squash(md)
|
|
97
|
+
if len(text) < 300 and len(md) < 300: # thin/empty page -> climb the ladder
|
|
98
|
+
d["blocked"] = True
|
|
99
|
+
d["error"] = "thin content (%d chars)" % len(text)
|
|
100
|
+
return d
|
|
101
|
+
d["title"] = body.get("title") or ""
|
|
102
|
+
d["text"] = text[:a.text_limit]
|
|
103
|
+
if a.out == "md":
|
|
104
|
+
d["markdown"] = md[:a.text_limit]
|
|
105
|
+
return d
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def find_launcher():
|
|
109
|
+
hit = shutil.which("crawl-env")
|
|
110
|
+
if hit:
|
|
111
|
+
return hit
|
|
112
|
+
if os.path.isfile(CRAWL_ENV_FALLBACK) and os.access(CRAWL_ENV_FALLBACK, os.X_OK):
|
|
113
|
+
return CRAWL_ENV_FALLBACK
|
|
114
|
+
return None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def main():
|
|
118
|
+
ap = argparse.ArgumentParser(description="tiered page fetch: DeepSeek Fetch API rung + container ladder")
|
|
119
|
+
ap.add_argument("--url", action="append", required=True)
|
|
120
|
+
ap.add_argument("--tier", default="auto",
|
|
121
|
+
choices=("auto", "deepseek", "static", "dynamic", "stealth"))
|
|
122
|
+
ap.add_argument("--no-deepseek", action="store_true",
|
|
123
|
+
help="skip the DeepSeek Fetch API rung in auto mode")
|
|
124
|
+
ap.add_argument("--timeout", type=int, default=45)
|
|
125
|
+
ap.add_argument("--out", default="json", choices=("json", "md"))
|
|
126
|
+
ap.add_argument("--text-limit", type=int, default=8000)
|
|
127
|
+
ap.add_argument("--proxy", default=None)
|
|
128
|
+
ap.add_argument("--persist", action="store_true")
|
|
129
|
+
ap.add_argument("--headful", action="store_true")
|
|
130
|
+
ap.add_argument("--host-net", action="store_true",
|
|
131
|
+
help="share host network so a loopback-only proxy is reachable")
|
|
132
|
+
a = ap.parse_args()
|
|
133
|
+
|
|
134
|
+
ok = blocked = False
|
|
135
|
+
leftover = list(a.url)
|
|
136
|
+
ds_key = getenv("DEEPSEEK_API_KEY")
|
|
137
|
+
if a.tier == "deepseek" or (a.tier == "auto" and ds_key and not a.no_deepseek):
|
|
138
|
+
leftover = []
|
|
139
|
+
for u in a.url:
|
|
140
|
+
d = deepseek_fetch(u, a, ds_key)
|
|
141
|
+
if d["blocked"] or d["error"]:
|
|
142
|
+
leftover.append(u)
|
|
143
|
+
blocked = blocked or d["blocked"]
|
|
144
|
+
if a.tier == "deepseek": # auto defers silently to the ladder
|
|
145
|
+
print(json.dumps(d, ensure_ascii=False))
|
|
146
|
+
continue
|
|
147
|
+
print(json.dumps(d, ensure_ascii=False))
|
|
148
|
+
ok = True
|
|
149
|
+
if a.tier == "deepseek":
|
|
150
|
+
return 0 if ok else (3 if blocked else 1)
|
|
151
|
+
|
|
152
|
+
if leftover:
|
|
153
|
+
launcher = find_launcher()
|
|
154
|
+
if launcher is None:
|
|
155
|
+
print("crawl-env 未安装:见 docs/fetching.md", file=sys.stderr)
|
|
156
|
+
return 0 if ok else 2
|
|
157
|
+
|
|
158
|
+
if not leftover:
|
|
159
|
+
return 0 if ok else (3 if blocked else 1)
|
|
160
|
+
|
|
161
|
+
inner = ["python3", PKG_DIR + "/fetch_container.py",
|
|
162
|
+
"--tier", "auto" if a.tier == "auto" else a.tier,
|
|
163
|
+
"--timeout", str(a.timeout),
|
|
164
|
+
"--out", a.out, "--text-limit", str(a.text_limit)]
|
|
165
|
+
if a.persist:
|
|
166
|
+
inner.append("--persist-container")
|
|
167
|
+
if a.headful:
|
|
168
|
+
inner.append("--headful-container")
|
|
169
|
+
# Only an explicit --proxy is forwarded: the shell's HTTPS_PROXY typically
|
|
170
|
+
# points at a host-loopback proxy the container cannot reach. crawl-env
|
|
171
|
+
# forwards CRAWL_PROXY itself (with loopback rewriting).
|
|
172
|
+
if a.proxy:
|
|
173
|
+
inner += ["--proxy", a.proxy]
|
|
174
|
+
for u in leftover:
|
|
175
|
+
inner += ["--url", u]
|
|
176
|
+
|
|
177
|
+
cmd = [launcher]
|
|
178
|
+
if a.persist:
|
|
179
|
+
cmd.append("--persist")
|
|
180
|
+
if a.headful:
|
|
181
|
+
cmd.append("--headful")
|
|
182
|
+
if a.host_net:
|
|
183
|
+
cmd.append("--host-net")
|
|
184
|
+
cmd += ["--ro-mount", PKG_DIR, "--"] + inner
|
|
185
|
+
|
|
186
|
+
try:
|
|
187
|
+
p = subprocess.run(cmd, capture_output=True, text=True, timeout=a.timeout * len(leftover) + 120)
|
|
188
|
+
except FileNotFoundError:
|
|
189
|
+
print("docker 不可用(crawl-env 依赖 docker)", file=sys.stderr)
|
|
190
|
+
return 0 if ok else 2
|
|
191
|
+
except subprocess.TimeoutExpired:
|
|
192
|
+
print("fetch 超时:容器未在预期时间内返回", file=sys.stderr)
|
|
193
|
+
return 0 if ok else 1
|
|
194
|
+
|
|
195
|
+
if p.returncode == 125 or "docker: command not found" in (p.stderr or ""):
|
|
196
|
+
print("docker 不可用:见 docs/fetching.md 故障排查", file=sys.stderr)
|
|
197
|
+
return 0 if ok else 2
|
|
198
|
+
if "no such image" in (p.stderr or "") or "Unable to find image" in (p.stderr or ""):
|
|
199
|
+
print("crawl-agent 镜像缺失:docker build -t crawl-agent:latest "
|
|
200
|
+
"~/workspace/crawl-agent", file=sys.stderr)
|
|
201
|
+
return 0 if ok else 2
|
|
202
|
+
|
|
203
|
+
for line in (p.stdout or "").splitlines():
|
|
204
|
+
line = line.strip()
|
|
205
|
+
if not line.startswith("{"):
|
|
206
|
+
continue
|
|
207
|
+
print(line)
|
|
208
|
+
try:
|
|
209
|
+
d = json.loads(line)
|
|
210
|
+
except json.JSONDecodeError:
|
|
211
|
+
continue
|
|
212
|
+
if d.get("error"):
|
|
213
|
+
continue
|
|
214
|
+
if d.get("blocked"):
|
|
215
|
+
blocked = True
|
|
216
|
+
else:
|
|
217
|
+
ok = True
|
|
218
|
+
if p.stderr:
|
|
219
|
+
print(p.stderr, file=sys.stderr, end="")
|
|
220
|
+
if ok:
|
|
221
|
+
return 0
|
|
222
|
+
return 3 if blocked else 1
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
if __name__ == "__main__":
|
|
226
|
+
sys.exit(main())
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""In-container fetch — runs ONLY inside crawl-agent (has scrapling + browsers).
|
|
3
|
+
|
|
4
|
+
Tier ladder on --tier auto (first hit wins):
|
|
5
|
+
static -> Fetcher (curl_cffi, TLS impersonation, no browser)
|
|
6
|
+
dynamic -> DynamicFetcher (playwright chromium, JS rendering)
|
|
7
|
+
stealth -> StealthyFetcher (patchright chromium, Cloudflare solving)
|
|
8
|
+
|
|
9
|
+
Prints one JSON line per URL on stdout (scrapling INFO logs go to stderr).
|
|
10
|
+
Exit 0 = >=1 ok; 1 = all failed; 3 = blocked on every URL (non-200, block
|
|
11
|
+
markers, or JS-shell thin content after the last tier).
|
|
12
|
+
"""
|
|
13
|
+
import argparse
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
|
|
19
|
+
BLOCK_MARKERS = (
|
|
20
|
+
"just a moment", "cf-challenge", "attention required",
|
|
21
|
+
"verify you are human", "enable javascript and cookies",
|
|
22
|
+
"access denied", "captcha", "人机验证", "请开启javascript",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def squash(s):
|
|
27
|
+
return re.sub(r"\s+", " ", s or "").strip()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def needs_escalation(status, html, text):
|
|
31
|
+
"""True when the page did not return real content (so the ladder climbs).
|
|
32
|
+
|
|
33
|
+
Thin-text heuristic catches JS shells: a 5 KB+ HTML skeleton rendering
|
|
34
|
+
under 300 chars of text is a client-rendered page, not an article.
|
|
35
|
+
"""
|
|
36
|
+
if status != 200:
|
|
37
|
+
return True
|
|
38
|
+
low = (html or "").lower()
|
|
39
|
+
if any(m in low for m in BLOCK_MARKERS):
|
|
40
|
+
return True
|
|
41
|
+
return len(text or "") < 300 and len(html or "") > 3000
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def result(url, tier_used, resp=None, include_md=False, error="", text_limit=8000):
|
|
45
|
+
d = {"url": url, "tier_used": tier_used, "status": None,
|
|
46
|
+
"title": "", "text": "", "markdown": "", "blocked": False,
|
|
47
|
+
"error": error}
|
|
48
|
+
if resp is None:
|
|
49
|
+
return d
|
|
50
|
+
d["status"] = resp.status
|
|
51
|
+
try:
|
|
52
|
+
d["title"] = squash(resp.css("title::text").get(default=""))
|
|
53
|
+
except Exception: # noqa: BLE001 — best-effort extraction, never aborts
|
|
54
|
+
pass
|
|
55
|
+
try:
|
|
56
|
+
d["text"] = squash(resp.get_all_text())[:text_limit]
|
|
57
|
+
except Exception: # noqa: BLE001
|
|
58
|
+
pass
|
|
59
|
+
if include_md:
|
|
60
|
+
try:
|
|
61
|
+
d["markdown"] = resp.markdown(main_content_only=True)[:text_limit]
|
|
62
|
+
except Exception: # noqa: BLE001
|
|
63
|
+
pass
|
|
64
|
+
return d
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def fetch_one(url, args):
|
|
68
|
+
from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher # noqa: PLC0415
|
|
69
|
+
|
|
70
|
+
tiers = {"static": ("static",), "dynamic": ("dynamic",),
|
|
71
|
+
"stealth": ("stealth",)}.get(args.tier,
|
|
72
|
+
("static", "dynamic", "stealth"))
|
|
73
|
+
explicit = args.proxy or os.environ.get("CRAWL_PROXY")
|
|
74
|
+
wildcard = os.environ.get("NO_PROXY") == "*" or os.environ.get("no_proxy") == "*"
|
|
75
|
+
# explicit CRAWL_PROXY/--proxy wins; otherwise wildcard NO_PROXY means direct;
|
|
76
|
+
# else fall back to the inherited HTTPS_PROXY.
|
|
77
|
+
proxy = explicit or (None if wildcard else os.environ.get("HTTPS_PROXY"))
|
|
78
|
+
headless = not (args.headful or args.headful_container)
|
|
79
|
+
last = None
|
|
80
|
+
for tier in tiers:
|
|
81
|
+
try:
|
|
82
|
+
if tier == "static":
|
|
83
|
+
kw = {"stealthy_headers": True, "impersonate": "chrome",
|
|
84
|
+
"timeout": args.timeout}
|
|
85
|
+
if proxy:
|
|
86
|
+
kw["proxy"] = proxy
|
|
87
|
+
resp = Fetcher.get(url, **kw)
|
|
88
|
+
elif tier == "dynamic":
|
|
89
|
+
kw = {"headless": headless, "network_idle": True,
|
|
90
|
+
"timeout": args.timeout * 1000, "extra_flags": ["--no-sandbox"],
|
|
91
|
+
"user_data_dir": "/tmp/home/chromium", "load_dom": True}
|
|
92
|
+
if proxy:
|
|
93
|
+
kw["proxy"] = proxy
|
|
94
|
+
resp = DynamicFetcher.fetch(url, **kw)
|
|
95
|
+
else:
|
|
96
|
+
kw = {"headless": headless, "solve_cloudflare": True,
|
|
97
|
+
"network_idle": True, "timeout": args.timeout * 1000,
|
|
98
|
+
"extra_flags": ["--no-sandbox"],
|
|
99
|
+
"user_data_dir": "/tmp/home/chromium"}
|
|
100
|
+
if proxy:
|
|
101
|
+
kw["proxy"] = proxy
|
|
102
|
+
resp = StealthyFetcher.fetch(url, **kw)
|
|
103
|
+
except Exception as e: # noqa: BLE001 — per-URL error lands in JSON
|
|
104
|
+
last = result(url, tier, error="%s: %s" % (type(e).__name__, e),
|
|
105
|
+
text_limit=args.text_limit)
|
|
106
|
+
continue
|
|
107
|
+
body = resp.body
|
|
108
|
+
html = body.decode("utf-8", "replace") if isinstance(body, bytes) else str(body)
|
|
109
|
+
d = result(url, tier, resp, include_md=(args.out == "md"),
|
|
110
|
+
text_limit=args.text_limit)
|
|
111
|
+
if not needs_escalation(resp.status, html, d["text"]):
|
|
112
|
+
return d
|
|
113
|
+
d["blocked"] = True
|
|
114
|
+
last = d
|
|
115
|
+
return last or result(url, None, error="no tier attempted",
|
|
116
|
+
text_limit=args.text_limit)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def main():
|
|
120
|
+
ap = argparse.ArgumentParser()
|
|
121
|
+
ap.add_argument("--url", action="append", required=True)
|
|
122
|
+
ap.add_argument("--tier", default="auto",
|
|
123
|
+
choices=("auto", "static", "dynamic", "stealth"))
|
|
124
|
+
ap.add_argument("--timeout", type=int, default=45)
|
|
125
|
+
ap.add_argument("--out", default="json", choices=("json", "md"))
|
|
126
|
+
ap.add_argument("--text-limit", type=int, default=8000)
|
|
127
|
+
ap.add_argument("--proxy", default=None)
|
|
128
|
+
ap.add_argument("--headful", action="store_true")
|
|
129
|
+
ap.add_argument("--headful-container", action="store_true")
|
|
130
|
+
ap.add_argument("--persist-container", action="store_true")
|
|
131
|
+
args = ap.parse_args()
|
|
132
|
+
|
|
133
|
+
n_ok, n_blocked = 0, 0
|
|
134
|
+
for u in args.url:
|
|
135
|
+
d = fetch_one(u, args)
|
|
136
|
+
print(json.dumps(d, ensure_ascii=False))
|
|
137
|
+
sys.stdout.flush()
|
|
138
|
+
if d["error"]:
|
|
139
|
+
continue
|
|
140
|
+
if d["blocked"]:
|
|
141
|
+
n_blocked += 1
|
|
142
|
+
else:
|
|
143
|
+
n_ok += 1
|
|
144
|
+
if n_ok:
|
|
145
|
+
return 0
|
|
146
|
+
return 3 if n_blocked else 1
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
if __name__ == "__main__":
|
|
150
|
+
sys.exit(main())
|