backlink-guard 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- backlink_guard/__init__.py +15 -0
- backlink_guard/__main__.py +181 -0
- backlink_guard/aisa.py +85 -0
- backlink_guard/blocklist.py +82 -0
- backlink_guard/cache.py +36 -0
- backlink_guard/checkout.py +160 -0
- backlink_guard/find.py +168 -0
- backlink_guard/goanyapi.py +53 -0
- backlink_guard/history.py +138 -0
- backlink_guard/ledger.py +71 -0
- backlink_guard/links.py +153 -0
- backlink_guard/lists.py +78 -0
- backlink_guard/lost.py +172 -0
- backlink_guard/page.py +158 -0
- backlink_guard/placement.py +55 -0
- backlink_guard/policy.py +164 -0
- backlink_guard/profile.py +210 -0
- backlink_guard/recheck.py +71 -0
- backlink_guard/report.py +297 -0
- backlink_guard/seen.py +45 -0
- backlink_guard/signals.py +167 -0
- backlink_guard/updates.py +93 -0
- backlink_guard/verify.py +80 -0
- backlink_guard/vet.py +697 -0
- backlink_guard-0.7.0.dist-info/METADATA +257 -0
- backlink_guard-0.7.0.dist-info/RECORD +30 -0
- backlink_guard-0.7.0.dist-info/WHEEL +5 -0
- backlink_guard-0.7.0.dist-info/entry_points.txt +2 -0
- backlink_guard-0.7.0.dist-info/licenses/LICENSE +21 -0
- backlink_guard-0.7.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""backlink-guard: check a link source before you submit to it, and watch your own link profile."""
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
__version__ = "0.7.0"
|
|
6
|
+
UA = f"Mozilla/5.0 (compatible; backlink-guard/{__version__}; +https://github.com/zhuyansen/backlink-guard)"
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def state_dir() -> Path:
|
|
10
|
+
"""Where the log, the cache and the networks seen before are kept: BACKLINK_GUARD_STATE when set, ./state when
|
|
11
|
+
it exists (a clone run from its own folder, as before 0.6), else ~/.backlink-guard, so an installed command
|
|
12
|
+
keeps one log wherever it is run from."""
|
|
13
|
+
if os.environ.get("BACKLINK_GUARD_STATE"):
|
|
14
|
+
return Path(os.environ["BACKLINK_GUARD_STATE"])
|
|
15
|
+
return Path("state") if Path("state").is_dir() else Path.home() / ".backlink-guard"
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""backlink-guard: check a link source before you submit to it, and watch your own links.
|
|
2
|
+
|
|
3
|
+
python -m backlink_guard find "<keyword>" [more ...] [--site yoursite.com] [--skip site|file] [--vet] [--limit 30] [--fresh] [--json]
|
|
4
|
+
python -m backlink_guard vet <site> [more ...] [--from list.xlsx|.csv|.md|.json] [--skip site|file] [--limit 30] [--topic "keyword"] [--referrals ga4.csv] [--deep] [--recent] [--free] [--fresh] [--json]
|
|
5
|
+
python -m backlink_guard recheck [--to your-domain] [--deep] [--free] [--fresh] [--json]
|
|
6
|
+
python -m backlink_guard audit <your-domain> [--deep] [--sitewide 20] [--rows 1000] [--fresh] [--json]
|
|
7
|
+
python -m backlink_guard lost <your-domain> [--sitewide 20] [--rows 1000] [--fresh] [--json]
|
|
8
|
+
python -m backlink_guard verify <page-url> --to <your-domain> [--json]
|
|
9
|
+
python -m backlink_guard log the record, with pace and anchor checks
|
|
10
|
+
python -m backlink_guard log add <site> --kind directory [--anchor TEXT] [--paid USD] [--url URL] [--note TEXT]
|
|
11
|
+
python -m backlink_guard log set <site> --status live [--url URL]
|
|
12
|
+
|
|
13
|
+
find the pages that rank for your keywords (directories, lists, articles, videos,
|
|
14
|
+
threads), sorted by the visitors they get from those searches, with what can be done
|
|
15
|
+
on each: $0.005 a keyword plus $0.012 for the search volumes. --vet then vets the sites.
|
|
16
|
+
vet reads each site, and with keys asks for its rating, a year of visits and its
|
|
17
|
+
spam score: about $0.01 a site plus $0.025 for the batch. --free reads the site only.
|
|
18
|
+
--from reads the sites out of a list file; --limit caps how many are checked.
|
|
19
|
+
--topic adds where each site ranks for your keyword ($0.005 a keyword).
|
|
20
|
+
--referrals reads your analytics export (GA4 Session source and Sessions) and shows
|
|
21
|
+
what each site has sent you. A site its own pages put at avoid gets no paid check.
|
|
22
|
+
recheck the sites in your log checked again and compared with the last look: what got
|
|
23
|
+
worse, and with --to, whether your links are still there. Monthly is enough.
|
|
24
|
+
audit your own profile: totals, new and lost referring domains by week, anchors with their
|
|
25
|
+
referring domains, site-wide links (one site linking from many of its pages, counted once),
|
|
26
|
+
and the spam score of the sites linking to you: about $0.11, with --deep about $0.15.
|
|
27
|
+
lost links to pages of yours that answer 404 or 410, by referring domains, each with the
|
|
28
|
+
closest live page in your sitemaps to 301 it to. The same backlinks as audit (about
|
|
29
|
+
$0.06, nothing more after an audit the same week); asking your pages is free.
|
|
30
|
+
verify whether a page links to you and whether that link passes ranking credit. Free.
|
|
31
|
+
log kept in state/ledger.jsonl on this machine. Free.
|
|
32
|
+
|
|
33
|
+
Keys: GOANYAPI_API_KEY (rating, visits, index, your backlinks) and AISA_API_KEY (spam
|
|
34
|
+
scores, weekly new and lost domains, anchors). Each is optional; what is missing is said.
|
|
35
|
+
"""
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import argparse
|
|
39
|
+
import json
|
|
40
|
+
import sys
|
|
41
|
+
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
|
|
44
|
+
from . import find as finding
|
|
45
|
+
from . import ledger, links, lists, lost, page, profile, recheck, report
|
|
46
|
+
from . import verify as verification
|
|
47
|
+
from . import vet as vetting
|
|
48
|
+
|
|
49
|
+
LIMIT = 30 # sites a run checks unless told otherwise: about $0.30 with both keys
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _print(args, data, text: str) -> int:
|
|
53
|
+
print(json.dumps(data, ensure_ascii=False, indent=1, default=sorted) if args.json else text)
|
|
54
|
+
return 0
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def skipped(args) -> set[str]:
|
|
58
|
+
return {page.site(x) for s in args.skip for x in (lists.read(s) if Path(s).is_file() else [s])}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def cmd_find(args) -> int:
|
|
62
|
+
result = finding.find(args.keywords, own=args.site or "", skip=tuple(skipped(args)), fresh=args.fresh)
|
|
63
|
+
vetted = {}
|
|
64
|
+
if args.vet:
|
|
65
|
+
sites = finding.to_vet(result, args.limit)
|
|
66
|
+
checked = vetting.vet(sites, fresh=args.fresh, topics=tuple(args.keywords)) if sites else {"rows": [], "cost": 0, "errors": []}
|
|
67
|
+
result["vet"], result["cost"] = checked, result["cost"] + checked["cost"]
|
|
68
|
+
vetted = {r["domain"]: r["grade"] for r in checked["rows"]}
|
|
69
|
+
text = report.find(result, vetted)
|
|
70
|
+
if args.vet and result["vet"]["rows"]:
|
|
71
|
+
text += "\n\n" + report.vet(result["vet"])
|
|
72
|
+
return _print(args, result, text)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def targets_of(args) -> list[str]:
|
|
76
|
+
"""The sites given and the sites in --from, once each, at most --limit of them."""
|
|
77
|
+
skip = skipped(args)
|
|
78
|
+
found = [t for t in dict.fromkeys(args.targets + (lists.read(args.from_file) if args.from_file else []))
|
|
79
|
+
if page.site(t) not in skip]
|
|
80
|
+
if not found:
|
|
81
|
+
raise SystemExit("vet: give sites, or a list with --from")
|
|
82
|
+
if len(found) > args.limit:
|
|
83
|
+
print(f"{len(found)} sites found; checking the first {args.limit} (raise --limit to check more)", file=sys.stderr)
|
|
84
|
+
return found[:args.limit]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def cmd_log(args) -> int:
|
|
88
|
+
if args.action == "add":
|
|
89
|
+
if not args.site or not args.kind:
|
|
90
|
+
raise SystemExit("log add: give the site and --kind")
|
|
91
|
+
ledger.add(args.site, args.kind, args.anchor or "", args.paid or 0.0, args.url or "", args.note or "")
|
|
92
|
+
elif args.action == "set":
|
|
93
|
+
if not ledger.update(args.site or "", status=args.status, url=args.url, note=args.note):
|
|
94
|
+
raise SystemExit(f"log set: no entry for {args.site}")
|
|
95
|
+
rows = ledger.entries()
|
|
96
|
+
flags = ledger.checks(rows)
|
|
97
|
+
return _print(args, {"entries": rows, "flags": flags}, report.ledger(rows, flags))
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def parser() -> argparse.ArgumentParser:
|
|
101
|
+
p = argparse.ArgumentParser(prog="python -m backlink_guard", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
102
|
+
sub = p.add_subparsers(dest="command", required=True)
|
|
103
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
104
|
+
common.add_argument("--json", action="store_true")
|
|
105
|
+
common.add_argument("--fresh", action="store_true", help="ignore answers cached in the last 7 days")
|
|
106
|
+
f = sub.add_parser("find", parents=[common])
|
|
107
|
+
f.add_argument("keywords", nargs="+", metavar="KEYWORD")
|
|
108
|
+
f.add_argument("--site", metavar="DOMAIN", help="your own site, left out of the results")
|
|
109
|
+
f.add_argument("--skip", action="append", default=[], metavar="SITE_OR_FILE",
|
|
110
|
+
help="leave out these sites (competitors, ones you already have); repeatable, or a list file")
|
|
111
|
+
f.add_argument("--vet", action="store_true", help="also vet the directories and article sites found (about $0.01 a site)")
|
|
112
|
+
f.add_argument("--limit", type=int, default=LIMIT, help=f"vet at most this many sites (default {LIMIT})")
|
|
113
|
+
v = sub.add_parser("vet", parents=[common])
|
|
114
|
+
v.add_argument("targets", nargs="*")
|
|
115
|
+
v.add_argument("--from", dest="from_file", metavar="FILE", help="a list to read sites from (.txt .md .csv .json .xlsx)")
|
|
116
|
+
v.add_argument("--limit", type=int, default=LIMIT, help=f"check at most this many sites (default {LIMIT})")
|
|
117
|
+
v.add_argument("--skip", action="append", default=[], metavar="SITE_OR_FILE",
|
|
118
|
+
help="leave out these sites (your own, competitors, ones you already have); repeatable, or a list file")
|
|
119
|
+
v.add_argument("--free", action="store_true", help="read the sites only; call no paid API")
|
|
120
|
+
v.add_argument("--deep", action="store_true",
|
|
121
|
+
help="also read each site's own backlink profile: badge networks, unproven sites (+$0.024 a site)")
|
|
122
|
+
v.add_argument("--topic", action="append", default=[], metavar="KEYWORD",
|
|
123
|
+
help="your keyword: shows where each site ranks for it (+$0.005 a keyword); repeatable")
|
|
124
|
+
v.add_argument("--recent", action="store_true",
|
|
125
|
+
help="also ask whether Google took in the site's pages from the last week and month (+$0.008 a site, slow)")
|
|
126
|
+
v.add_argument("--referrals", metavar="FILE",
|
|
127
|
+
help="your analytics export of visitor sources (GA4 CSV: Session source, Sessions): shows what each site sent you")
|
|
128
|
+
r = sub.add_parser("recheck", parents=[common])
|
|
129
|
+
r.add_argument("--to", metavar="DOMAIN", default="", help="your domain: also check that your links are still there (free)")
|
|
130
|
+
r.add_argument("--deep", action="store_true", help="also the backlink profile of each site (+$0.024 a site)")
|
|
131
|
+
r.add_argument("--free", action="store_true", help="read the sites only; call no paid API")
|
|
132
|
+
a = sub.add_parser("audit", parents=[common])
|
|
133
|
+
a.add_argument("domain")
|
|
134
|
+
a.add_argument("--deep", action="store_true", help="also the full anchor distribution")
|
|
135
|
+
o = sub.add_parser("lost", parents=[common])
|
|
136
|
+
o.add_argument("domain")
|
|
137
|
+
for x in (a, o):
|
|
138
|
+
x.add_argument("--sitewide", type=int, default=links.SITEWIDE_PAGES, metavar="PAGES",
|
|
139
|
+
help=f"pages of one site linking to one page of yours that make a site-wide link (default {links.SITEWIDE_PAGES})")
|
|
140
|
+
x.add_argument("--rows", type=int, default=links.ROWS, metavar="LINKS",
|
|
141
|
+
help=f"backlinks read one by one from AIsa (default and most {links.ROWS}; about $0.06 for 1,000)")
|
|
142
|
+
c = sub.add_parser("verify", parents=[common])
|
|
143
|
+
c.add_argument("page")
|
|
144
|
+
c.add_argument("--to", required=True, metavar="DOMAIN")
|
|
145
|
+
g = sub.add_parser("log", parents=[common])
|
|
146
|
+
g.add_argument("action", nargs="?", choices=["add", "set"])
|
|
147
|
+
g.add_argument("site", nargs="?")
|
|
148
|
+
g.add_argument("--kind", choices=ledger.KINDS)
|
|
149
|
+
g.add_argument("--status", choices=ledger.STATUSES)
|
|
150
|
+
for name in ("--anchor", "--url", "--note"):
|
|
151
|
+
g.add_argument(name)
|
|
152
|
+
g.add_argument("--paid", type=float)
|
|
153
|
+
return p
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def main(argv: list[str] | None = None) -> int:
|
|
157
|
+
args = parser().parse_args(argv)
|
|
158
|
+
if args.command == "vet":
|
|
159
|
+
targets = targets_of(args)
|
|
160
|
+
result = vetting.vet(targets, paid_sources=not args.free, fresh=args.fresh, recent=args.recent, deep=args.deep,
|
|
161
|
+
topics=tuple(args.topic), referrals=lists.referrals(args.referrals) if args.referrals else None)
|
|
162
|
+
return _print(args, result, report.vet(result))
|
|
163
|
+
if args.command == "recheck":
|
|
164
|
+
result = recheck.run(own=args.to, paid_sources=not args.free, deep=args.deep, fresh=args.fresh)
|
|
165
|
+
return _print(args, result, report.recheck(result))
|
|
166
|
+
if args.command == "find":
|
|
167
|
+
return cmd_find(args)
|
|
168
|
+
if args.command == "audit":
|
|
169
|
+
result = profile.audit(args.domain, deep=args.deep, fresh=args.fresh, sitewide=args.sitewide, rows=args.rows)
|
|
170
|
+
return _print(args, result, report.audit(result))
|
|
171
|
+
if args.command == "lost":
|
|
172
|
+
result = lost.lost(args.domain, fresh=args.fresh, rows=args.rows, sitewide=args.sitewide)
|
|
173
|
+
return _print(args, result, report.lost(result))
|
|
174
|
+
if args.command == "verify":
|
|
175
|
+
result = verification.verify(args.page, args.to)
|
|
176
|
+
return _print(args, result, report.verify(result))
|
|
177
|
+
return cmd_log(args)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
if __name__ == "__main__":
|
|
181
|
+
sys.exit(main())
|
backlink_guard/aisa.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""AIsa's tool router: backlink data (spam score, new and lost referring
|
|
2
|
+
domains, anchors). Key: AISA_API_KEY, or the one the AIsa CLI stores at login.
|
|
3
|
+
|
|
4
|
+
Every call is billed and none is retried here. Prices seen on 2026-10-13: spam score for
|
|
5
|
+
14 domains in one call $0.025, a new/lost timeline $0.025, anchors about $0.04.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import urllib.error
|
|
12
|
+
import urllib.request
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from . import UA
|
|
16
|
+
|
|
17
|
+
ROUTER = os.environ.get("AISA_ROUTER_BASE_URL", "https://tools.aisa.one") + "/v1/tool-router/aisa-batch-use"
|
|
18
|
+
# AIsa's passthrough to the same endpoints, as they are: the router's schema refuses their array
|
|
19
|
+
# filters ("got array, want null or object", 2026-10-15), this path takes them.
|
|
20
|
+
REST = os.environ.get("AISA_API_BASE_URL", "https://api.aisa.one") + "/apis/v1/dataforseo/"
|
|
21
|
+
TIMEOUT = 240
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class AisaError(RuntimeError):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def key() -> str:
|
|
29
|
+
""""" when there is no key: callers skip this source, they do not fail."""
|
|
30
|
+
if os.environ.get("AISA_API_KEY"):
|
|
31
|
+
return os.environ["AISA_API_KEY"].strip()
|
|
32
|
+
stored = Path.home() / ".aisa" / "key"
|
|
33
|
+
return stored.read_text().strip() if stored.exists() else ""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _run(tool: str, arguments: dict) -> tuple[object, int]:
|
|
37
|
+
"""One call through the tool router: (its data, cost in micro-USD)."""
|
|
38
|
+
payload = {"calls": [{"call_id": "c", "tool": tool, "arguments": arguments}]}
|
|
39
|
+
req = urllib.request.Request(ROUTER, data=json.dumps(payload).encode(), method="POST", headers={
|
|
40
|
+
"Content-Type": "application/json", "Authorization": f"Bearer {key()}", "User-Agent": UA,
|
|
41
|
+
"x-aisa-source": "backlink-guard"})
|
|
42
|
+
try:
|
|
43
|
+
with urllib.request.urlopen(req, timeout=TIMEOUT) as res:
|
|
44
|
+
reply = json.load(res)
|
|
45
|
+
except urllib.error.HTTPError as exc:
|
|
46
|
+
raise AisaError(f"HTTP {exc.code}: {' '.join(exc.read().decode('utf-8', 'replace').split())[:160]}") from exc
|
|
47
|
+
except (OSError, ValueError) as exc:
|
|
48
|
+
raise AisaError(f"{type(exc).__name__}: {exc}") from exc
|
|
49
|
+
row = (reply.get("results") or [{}])[0]
|
|
50
|
+
if "error" in reply or not row.get("successful"):
|
|
51
|
+
raise AisaError(str(reply.get("error") or row.get("error") or "call failed")[:160])
|
|
52
|
+
return row.get("data"), int(row.get("customer_cost_micros_usd") or 0)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def tool(name: str, arguments: dict) -> tuple[object, int]:
|
|
56
|
+
"""A tool whose answer is not a task list (Ahrefs, Semrush): its data as the router returns it."""
|
|
57
|
+
return _run(name, arguments)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def call(tool: str, body: dict) -> tuple[dict, int]:
|
|
61
|
+
"""One AIsa task. Returns (the task's first result, cost in micro-USD)."""
|
|
62
|
+
data, cost = _run(tool, {"body": [body]})
|
|
63
|
+
task = (((data or {}) if isinstance(data, dict) else {}).get("tasks") or [{}])[0]
|
|
64
|
+
if not task.get("result"):
|
|
65
|
+
raise AisaError(str(task.get("status_message") or "no result")[:160])
|
|
66
|
+
return task["result"][0] or {}, cost
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def rest(path: str, body: dict) -> tuple[dict, int]:
|
|
70
|
+
"""One live task through AIsa's passthrough: (its first result, cost in micro-USD)."""
|
|
71
|
+
req = urllib.request.Request(REST + path, data=json.dumps([body]).encode(), method="POST", headers={
|
|
72
|
+
"Content-Type": "application/json", "Authorization": f"Bearer {key()}", "User-Agent": UA,
|
|
73
|
+
"x-aisa-source": "backlink-guard"})
|
|
74
|
+
try:
|
|
75
|
+
with urllib.request.urlopen(req, timeout=TIMEOUT) as res:
|
|
76
|
+
reply = json.load(res)
|
|
77
|
+
except urllib.error.HTTPError as exc:
|
|
78
|
+
raise AisaError(f"HTTP {exc.code}: {' '.join(exc.read().decode('utf-8', 'replace').split())[:160]}") from exc
|
|
79
|
+
except (OSError, ValueError) as exc:
|
|
80
|
+
raise AisaError(f"{type(exc).__name__}: {exc}") from exc
|
|
81
|
+
task = (reply.get("tasks") or [{}])[0]
|
|
82
|
+
if reply.get("status_code") != 20000 or task.get("status_code") != 20000:
|
|
83
|
+
raise AisaError(str(task.get("status_message") or reply.get("status_message") or "call failed")[:160])
|
|
84
|
+
cost = float(reply.get("cost") or task.get("cost") or 0)
|
|
85
|
+
return (task.get("result") or [{}])[0] or {}, int(round(cost * 1e6))
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""The shared list of sites that build or sell spam backlinks, each with public evidence
|
|
2
|
+
(github.com/zhuyansen/backlink-spam-list). A site listed there is avoid, with the entry's evidence.
|
|
3
|
+
Read once a run and kept a day; free. A list that cannot be read is reported, never read as empty.
|
|
4
|
+
The list grows by reports: a check that finds one operator's sites missing from it links to a filled-in report.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import csv
|
|
9
|
+
import io
|
|
10
|
+
import os
|
|
11
|
+
import urllib.request
|
|
12
|
+
from datetime import date
|
|
13
|
+
from urllib.parse import quote, urlencode
|
|
14
|
+
|
|
15
|
+
from . import UA, __version__, cache
|
|
16
|
+
from .policy import flag
|
|
17
|
+
|
|
18
|
+
URL = os.environ.get("BACKLINK_SPAM_LIST_URL", "https://raw.githubusercontent.com/zhuyansen/backlink-spam-list/main/list.csv")
|
|
19
|
+
REPORT_FORM = os.environ.get("BACKLINK_SPAM_LIST_REPORT", "https://github.com/zhuyansen/backlink-spam-list/issues/new")
|
|
20
|
+
NETWORK = "link-network (one operator's sites - a shared template, redirect, checkout link or account)" # the form's option
|
|
21
|
+
TTL_DAYS = 1
|
|
22
|
+
TIMEOUT = 20
|
|
23
|
+
REPORT_MIN = 3 # sites one operator's group needs before it is worth a report
|
|
24
|
+
REPORT_SITES = 30 # sites one report carries at most
|
|
25
|
+
OPERATOR = {"same_template", "same_owner"} # findings that only say one operator runs the sites
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def load(fresh: bool = False) -> dict[str, dict]:
|
|
29
|
+
"""{domain: entry} for the entries still listed. Raises OSError or ValueError when the list cannot be read."""
|
|
30
|
+
held = None if fresh else cache.get(f"spamlist:{URL}", ttl_days=TTL_DAYS)
|
|
31
|
+
if held is not None:
|
|
32
|
+
return held
|
|
33
|
+
with urllib.request.urlopen(urllib.request.Request(URL, headers={"User-Agent": UA}), timeout=TIMEOUT) as res:
|
|
34
|
+
text = res.read().decode("utf-8")
|
|
35
|
+
rows = csv.DictReader(io.StringIO(text))
|
|
36
|
+
held = {r["domain"]: {k: r.get(k, "") for k in ("category", "family", "evidence", "evidence_url", "last_checked")}
|
|
37
|
+
for r in rows if r.get("status") == "listed"}
|
|
38
|
+
cache.put(f"spamlist:{URL}", held)
|
|
39
|
+
return held
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def flag_for(domain: str, listed: dict[str, dict]) -> dict | None:
|
|
43
|
+
e = listed.get(domain)
|
|
44
|
+
if not e:
|
|
45
|
+
return None
|
|
46
|
+
family = f", the {e['family']} family" if e.get("family") else ""
|
|
47
|
+
return flag("listed", f"{e['category']}{family}: {e['evidence'][:180]} ({e['evidence_url']}, checked {e['last_checked']})")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def against(row: dict) -> bool:
|
|
51
|
+
"""Something stands against the site besides sharing an operator, which a company's honest sites also do."""
|
|
52
|
+
return any(f["level"] in ("avoid", "caution") and f["flag"] not in OPERATOR for f in row["flags"])
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def unlisted(groups: list[dict], rows: list[dict], listed: dict[str, dict] | None, today: date) -> list[dict]:
|
|
56
|
+
"""The groups of one operator's sites (vet.shared_templates, vet.shared_owners) worth a report: REPORT_MIN sites
|
|
57
|
+
or more, at least half of those in this check with a finding against them besides the shared operator, and
|
|
58
|
+
some not listed yet. Each with the link to a filled-in report. None when the list could not be read."""
|
|
59
|
+
if listed is None:
|
|
60
|
+
return []
|
|
61
|
+
by_site, out = {r["domain"]: r for r in rows}, []
|
|
62
|
+
for g in groups:
|
|
63
|
+
checked = [d for d in g["sites"] if d in by_site]
|
|
64
|
+
flagged = [d for d in checked if against(by_site[d])]
|
|
65
|
+
missing = [d for d in g["sites"] if d not in listed]
|
|
66
|
+
if len(g["sites"]) >= REPORT_MIN and missing and 2 * len(flagged) >= len(checked):
|
|
67
|
+
out.append({**g, "unlisted": missing, "report": report_url(g, missing, today)})
|
|
68
|
+
return out
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def report_url(group: dict, sites: list[str], today: date) -> str:
|
|
72
|
+
"""The list's report form, filled in: the first site, the others as members, what they share. Opening it sends
|
|
73
|
+
nothing; the reporter reads it and submits, and a maintainer checks the evidence before anything is listed."""
|
|
74
|
+
shown = sites[:REPORT_SITES]
|
|
75
|
+
shared = (f"{len(group['sites'])} sites share one page title, each site's own name taken out: \"{group['key']}\""
|
|
76
|
+
if group["kind"] == "template" else
|
|
77
|
+
f"{len(group['sites'])} sites report to one analytics or ads account: {group['key']}")
|
|
78
|
+
fields = {"template": "report.yml", "title": f"Report: {shown[0]}" + (f" and {len(shown) - 1} more" if len(shown) > 1 else ""),
|
|
79
|
+
"domain": shown[0], "category": NETWORK, "evidence-url": f"https://{shown[0]}/", "evidence": shared,
|
|
80
|
+
"members": "\n".join(shown[1:]),
|
|
81
|
+
"context": f"Found with backlink-guard {__version__} on {today.isoformat()}: the sites came up in checks together."}
|
|
82
|
+
return f"{REPORT_FORM}?{urlencode(fields, quote_via=quote)}"
|
backlink_guard/cache.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Paid answers kept on disk for a few days, so checking the same site twice in a week
|
|
2
|
+
is paid for once. One JSON file per question under <state>/cache/."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import time
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from . import state_dir
|
|
11
|
+
|
|
12
|
+
DAY = 86400
|
|
13
|
+
TTL_DAYS = 7
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _dir() -> Path:
|
|
17
|
+
return state_dir() / "cache"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _path(key: str) -> Path:
|
|
21
|
+
return _dir() / (hashlib.sha1(key.encode()).hexdigest()[:20] + ".json")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def get(key: str, ttl_days: float = TTL_DAYS):
|
|
25
|
+
path = _path(key)
|
|
26
|
+
if not path.exists() or time.time() - path.stat().st_mtime > ttl_days * DAY:
|
|
27
|
+
return None
|
|
28
|
+
try:
|
|
29
|
+
return json.loads(path.read_text())
|
|
30
|
+
except ValueError:
|
|
31
|
+
return None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def put(key: str, value) -> None:
|
|
35
|
+
_dir().mkdir(parents=True, exist_ok=True)
|
|
36
|
+
_path(key).write_text(json.dumps(value, ensure_ascii=False))
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""Checkout links shared across sites (0.5, from the payment-link history of 2026-10-15).
|
|
2
|
+
|
|
3
|
+
A checkout link (a Stripe Payment Link, a Gumroad product, Lemon Squeezy, Polar, Creem, Dodo or
|
|
4
|
+
Paddle) belongs to one seller. When the one a site carries also sits on many other sites, either
|
|
5
|
+
one operator's network sells through it or it was injected into hacked sites. On 2026-10-15 one
|
|
6
|
+
Stripe link sat on 395 sites titled like "Backlink Generator Tool" and "SEO Score Checker", and a
|
|
7
|
+
Gumroad link for an escort service had been injected into 262 schools, clinics and museums. A
|
|
8
|
+
product people write about is shared too (MacWhisper's link sat on 705 sites), by sites that look
|
|
9
|
+
nothing alike: that is not flagged.
|
|
10
|
+
|
|
11
|
+
Network sites put their checkout links on a few of thousands of inner pages (/dir/<slug>-<id>),
|
|
12
|
+
which no sample of pages finds (six sitemap pages of two network sites held none). So the home page
|
|
13
|
+
is read for free, and with --deep AIsa also lists the site's links to Stripe and Gumroad, the
|
|
14
|
+
platforms of nearly every shared link (one call each, about $0.024). One more call per checkout link
|
|
15
|
+
found (about $0.02), at most LOOKED_AT per site; most link sources carry none and cost little.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import re
|
|
20
|
+
from collections import Counter
|
|
21
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
22
|
+
|
|
23
|
+
from . import aisa, page
|
|
24
|
+
from .policy import flag
|
|
25
|
+
|
|
26
|
+
TOOL = "post_dataforseo_backlinks_backlinks_live"
|
|
27
|
+
CHECKOUT = re.compile(r"https?://(?:buy\.stripe\.com/[A-Za-z0-9_]+|(?:[a-z0-9-]+\.)?gumroad\.com/l/[^/?#\s]+|"
|
|
28
|
+
r"[a-z0-9-]+\.lemonsqueezy\.com/(?:checkout/)?buy/[^/?#\s]+|buy\.polar\.sh/[^/?#\s]+|"
|
|
29
|
+
r"(?:www\.)?creem\.io/payment/[^/?#\s]+|checkout\.dodopayments\.com/buy/[^/?#\s]+|"
|
|
30
|
+
r"buy\.paddle\.com/[^?#\s]+)", re.I)
|
|
31
|
+
LOOKED_AT = 2 # checkout links looked up per site
|
|
32
|
+
LIMIT = 100 # other sites read per checkout link
|
|
33
|
+
SHARED_MIN = 5 # other sites carrying the same link before it means anything
|
|
34
|
+
TEMPLATE_SHARE = 0.4 # this share of those sites' titles alike (same ending, or about selling links)
|
|
35
|
+
TAIL_WORDS = 2 # "drjack.world | Keyword Rankingss": the site's name sits right before the template's words
|
|
36
|
+
TITLES_READ = 24 # AIsa's rows leave most page titles empty: this many home pages are read for theirs
|
|
37
|
+
MIRRORS_MAX = 20 # up to this many sites with one identical title can be one site on several domains (17 seen)
|
|
38
|
+
INJECTED_RELATED = 0.3 # a gambling or adult product cited by sites on its own topic is promotion, not injection
|
|
39
|
+
PLATFORMS = ("stripe.com", "gumroad.com") # with --deep: where the site's own checkout links are looked for
|
|
40
|
+
LINKED = 20 # of the site's links to each
|
|
41
|
+
# What the network sites of 2026-10-15 were titled: SEO tools and link sales, one template per site.
|
|
42
|
+
LINK_TRADE = re.compile(r"\bbacklinks?\b|\blink building\b|\bguest posts?\b|\bkeyword ranking|\bchecker\b|\bpbn\b|"
|
|
43
|
+
r"\b(?:traffic|backlink) generator\b|\bda\s*pa\b|\baged domains?\b|\bdomain authority\b", re.I) # not "seo": SEO blogs cite SEO courses
|
|
44
|
+
# Words of the products injected into hacked sites, besides the gambling and adult words vet already has.
|
|
45
|
+
INJECTED = re.compile(r"\b(?:travesti|bahis|bokep|jav|call ?girls?|escorts?)\b", re.I)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def links_on(links: list[dict]) -> list[str]:
|
|
49
|
+
"""The checkout links among `links`, query strings dropped, the first LOOKED_AT in page order."""
|
|
50
|
+
found: list[str] = []
|
|
51
|
+
for link in links:
|
|
52
|
+
m = CHECKOUT.match(link["href"])
|
|
53
|
+
url = m.group(0).rstrip("/") if m else ""
|
|
54
|
+
if url and url not in found:
|
|
55
|
+
found.append(url)
|
|
56
|
+
return found[:LOOKED_AT]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def linked(site: str, ask) -> tuple[list[dict], int]:
|
|
60
|
+
"""With --deep: the site's links to PLATFORMS, as page links ({"href"}), and the cost."""
|
|
61
|
+
found, cost = [], 0
|
|
62
|
+
for target in PLATFORMS:
|
|
63
|
+
data, spent = ask(f"checkout-from:{site}:{target}", lambda target=target: aisa.rest("backlinks/backlinks/live", {
|
|
64
|
+
"target": target, "mode": "as_is", "limit": LINKED, "backlinks_status_type": "live",
|
|
65
|
+
"filters": [["domain_from", "like", f"%{site}"]]}))
|
|
66
|
+
found += [{"href": i.get("url_to") or ""} for i in (data or {}).get("items") or []]
|
|
67
|
+
cost += spent
|
|
68
|
+
return found, cost
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def others(url: str, own: str, ask) -> tuple[list[dict], int]:
|
|
72
|
+
"""The other sites linking to `url`: [{"site", "title"}] (one row a site), and the cost."""
|
|
73
|
+
try:
|
|
74
|
+
data, cost = ask(f"checkout:{url}", lambda: aisa.call(TOOL, {
|
|
75
|
+
"target": url, "mode": "one_per_domain", "limit": LIMIT, "backlinks_status_type": "live"}))
|
|
76
|
+
except aisa.AisaError as exc:
|
|
77
|
+
if "no result" in str(exc).lower(): # nobody links to it: nothing shared
|
|
78
|
+
return [], 0
|
|
79
|
+
raise
|
|
80
|
+
rows, seen = [], {own}
|
|
81
|
+
for item in (data or {}).get("items") or []:
|
|
82
|
+
site = page.site("https://" + (item.get("domain_from") or ""))
|
|
83
|
+
if site and site not in seen:
|
|
84
|
+
seen.add(site)
|
|
85
|
+
rows.append({"site": site, "title": item.get("page_from_title") or "", "host": item.get("domain_from") or site})
|
|
86
|
+
return titled(rows), cost
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def titled(rows: list[dict]) -> list[dict]:
|
|
90
|
+
"""Rows with a title: the ones AIsa left empty read from their home pages, TITLES_READ at most."""
|
|
91
|
+
blank = [r for r in rows if not r["title"].strip()][:TITLES_READ]
|
|
92
|
+
with ThreadPoolExecutor(12) as pool:
|
|
93
|
+
for r, got in zip(blank, pool.map(lambda r: page.fetch(f"https://{r['host']}/"), blank)):
|
|
94
|
+
r["title"] = page.parse(got["html"], got["final"])["title"] if got["html"] else ""
|
|
95
|
+
return rows
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def tail(title: str) -> str:
|
|
99
|
+
words = re.findall(r"[a-z0-9]+", title.lower())
|
|
100
|
+
return " ".join(words[-TAIL_WORDS:]) if len(words) > TAIL_WORDS else ""
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def judge(url: str, own_title: str, rows: list[dict], spam: re.Pattern) -> dict | None:
|
|
104
|
+
"""A flag when the site is one of many carrying `url` and is like them: titled about link sales, or built
|
|
105
|
+
from the same template, or one of the unrelated sites a spam product was injected into. A site whose
|
|
106
|
+
link was merely copied onto such sites (the seller's own, or one that cites the product) is not flagged.
|
|
107
|
+
Counted by site: on 2026-10-15, 94-99 of the 100 sites carrying each of three network links shared one
|
|
108
|
+
title ("Expert backlink building services for higher Google rankings..."). A few sites with one identical
|
|
109
|
+
title are more likely one site on several domains, which a template needs more of (MIRRORS_MAX) to count."""
|
|
110
|
+
n = len(rows)
|
|
111
|
+
if n < SHARED_MIN:
|
|
112
|
+
return None
|
|
113
|
+
count = f"{n}{'+' if n >= LIMIT else ''}"
|
|
114
|
+
examples = ", ".join(r["site"] for r in rows[:3])
|
|
115
|
+
own = " ".join(own_title.lower().split())
|
|
116
|
+
titles = [" ".join(r["title"].lower().split()) for r in rows if r["title"].strip()]
|
|
117
|
+
need = max(SHARED_MIN, TEMPLATE_SHARE * len(titles))
|
|
118
|
+
if spam.search(url) or INJECTED.search(url):
|
|
119
|
+
related = sum(1 for t in titles if spam.search(t) or INJECTED.search(t))
|
|
120
|
+
if related < INJECTED_RELATED * len(titles) and not (spam.search(own) or INJECTED.search(own)):
|
|
121
|
+
return flag("injected_checkout", f"{url} also sits on {count} other unrelated sites, e.g. {examples}")
|
|
122
|
+
return None
|
|
123
|
+
trade = sum(1 for t in titles if LINK_TRADE.search(t))
|
|
124
|
+
if trade >= need and LINK_TRADE.search(own) and len(set(titles)) > 1 or trade >= max(need, MIRRORS_MAX + 1) and LINK_TRADE.search(own):
|
|
125
|
+
return flag("checkout_network", f"{url} also sits on {count} other sites; {trade} of them are titled about "
|
|
126
|
+
f"backlinks, link checkers or link sales, like this one, e.g. {examples}")
|
|
127
|
+
by_tail: dict[str, list[str]] = {}
|
|
128
|
+
for t in titles:
|
|
129
|
+
if tail(t):
|
|
130
|
+
by_tail.setdefault(tail(t), []).append(t)
|
|
131
|
+
ending, group = max(by_tail.items(), key=lambda kv: len(kv[1]), default=("", []))
|
|
132
|
+
template = len(set(group)) >= 3 or len(group) > MIRRORS_MAX
|
|
133
|
+
if len(group) >= need and template and tail(own) == ending:
|
|
134
|
+
return flag("checkout_network", f"{url} also sits on {count} other sites; {len(group)} of their titles end "
|
|
135
|
+
f"\"…{ending}\" like this one's, e.g. {examples}")
|
|
136
|
+
return None
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def shared(read: dict, own: str, ask, spam: re.Pattern, deep: bool = False) -> tuple[list[dict], int, list[str]]:
|
|
140
|
+
"""(flags, cost, errors) for the checkout links on a site read by vet.read_site; with `deep`, also the ones
|
|
141
|
+
AIsa saw on its other pages."""
|
|
142
|
+
flags, cost, errors = [], 0, []
|
|
143
|
+
links = list(read["parsed"]["links"])
|
|
144
|
+
if deep:
|
|
145
|
+
try:
|
|
146
|
+
more, cost = linked(own, ask)
|
|
147
|
+
links += more
|
|
148
|
+
except aisa.AisaError as exc:
|
|
149
|
+
errors.append(f"checkout links: {exc}")
|
|
150
|
+
for url in links_on(links):
|
|
151
|
+
try:
|
|
152
|
+
rows, spent = others(url, own, ask)
|
|
153
|
+
except aisa.AisaError as exc:
|
|
154
|
+
errors.append(f"checkout link {url}: {exc}")
|
|
155
|
+
continue
|
|
156
|
+
cost += spent
|
|
157
|
+
found = judge(url, read["parsed"]["title"], rows, spam)
|
|
158
|
+
if found and found["flag"] not in {f["flag"] for f in flags}: # one flag of a kind, the first link's evidence
|
|
159
|
+
flags.append(found)
|
|
160
|
+
return flags, cost, errors
|