paytrace 2.0.2__tar.gz → 2.0.4__tar.gz

{paytrace-2.0.2 → paytrace-2.0.4}/CHANGELOG.md

@@ -6,6 +6,26 @@ versioning follows [SemVer](https://semver.org/).
 
 ## [Unreleased]
 
+## [2.0.4] - 2026-09-23
+
+- sellers.json documents larger than the body cap are now streamed and scanned
+  instead of truncated. Google's is 104 MB uncompressed, so the payee behind any
+  Google-monetised site could never be named — the ad system serving most
+  ad-funded sites. Only the matched record is kept; the retrieval stays
+  evidenced by a byte count and a SHA-256 taken over the stream.
+- Scan-mode retrievals bypass the cache, which would otherwise serve the
+  truncated copy left by an earlier ordinary fetch.
+
+## [2.0.3] - 2026-09-23
+
+- One AdSense payee is now one node. `ca-pub-N` in page source and `pub-N` in
+  ads.txt were captured in different shapes by different collectors, so a single
+  account became two graph nodes with its evidence divided between them — on a
+  real domain the same ID resolved STRONG_EVIDENCE under one spelling and
+  UNSUPPORTED under the other. The `adsense_pub` scheme is gone with it.
+- UA identifiers now key on the account (`UA-1234`), not the property
+  (`UA-1234-2`), in every collector.
+
 ## [2.0.2] - 2026-09-22
 
 - `paytrace run` no longer crashes on a real network. It closed its HTTP

{paytrace-2.0.2 → paytrace-2.0.4}/CITATION.cff

@@ -25,4 +25,4 @@ keywords:
   - attribution
   - corporate registries
 license: Apache-2.0
-version: 2.0.2
+version: 2.0.4

{paytrace-2.0.2 → paytrace-2.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: paytrace
-Version: 2.0.2
+Version: 2.0.4
 Summary: Trace a website to the legal entity that gets paid for it
 Project-URL: Homepage, https://github.com/tusharkarumudi/paytrace
 Project-URL: Issues, https://github.com/tusharkarumudi/paytrace/issues

{paytrace-2.0.2 → paytrace-2.0.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "paytrace"
-version = "2.0.2"
+version = "2.0.4"
 description = "Trace a website to the legal entity that gets paid for it"
 readme = "README.md"
 requires-python = ">=3.11"

{paytrace-2.0.2 → paytrace-2.0.4}/src/paytrace/_version.py

@@ -5,4 +5,4 @@ circular import: the package __init__ imports those submodules before it has
 finished defining __version__.
 """
 
-__version__ = "2.0.2"
+__version__ = "2.0.4"

{paytrace-2.0.2 → paytrace-2.0.4}/src/paytrace/collectors/analytics.py

@@ -37,10 +37,12 @@ from .base import Collector, register
 # above infrastructure.
 
 ID_PATTERNS: dict[str, tuple[re.Pattern, Reliability]] = {
-    # ca-pub-XXXXXXXXXXXXXXXX -- one AdSense payee account
-    "adsense": (re.compile(r"\bca-pub-(\d{16})\b"), Reliability.AUTHORITATIVE),
-    # pub-XXXXXXXXXXXXXXXX in ads.txt context
-    "adsense_pub": (re.compile(r"\bpub-(\d{16})\b"), Reliability.AUTHORITATIVE),
+    # One AdSense payee account, written `ca-pub-N` in page source and `pub-N`
+    # in ads.txt. ONE pattern, capturing the digits: two patterns under two
+    # scheme names split one payee into separate graph nodes, so its evidence
+    # was divided and the same ID could come out STRONG_EVIDENCE under one
+    # spelling and UNSUPPORTED under the other.
+    "adsense": (re.compile(r"\b(?:ca-)?pub-(\d{16})\b"), Reliability.AUTHORITATIVE),
     # G-XXXXXXXXXX GA4 measurement
     "ga4": (re.compile(r"\b(G-[A-Z0-9]{8,12})\b"), Reliability.STRONG),
     # UA-XXXXXXXX-N legacy Universal Analytics; the account part is UA-XXXXXXXX

{paytrace-2.0.2 → paytrace-2.0.4}/src/paytrace/collectors/artifacts.py

@@ -76,10 +76,14 @@ _STAGING = re.compile(
 
 #: Service identifier schemes worth extracting from source. Each is a pivot.
 _SERVICE_IDS = {
-    "adsense": re.compile(r"\b(pub-\d{16})\b"),
+    # Digits only, matching analytics.py: the `pub-` prefix is spelling, not
+    # identity. Keeping it here made one payee two nodes.
+    "adsense": re.compile(r"\b(?:ca-)?pub-(\d{16})\b"),
     "ga4": re.compile(r"\b(G-[A-Z0-9]{8,12})\b"),
-    "ua": re.compile(r"\b(UA-\d{4,10}-\d{1,4})\b"),
-    "gtm": re.compile(r"\b(GTM-[A-Z0-9]{6,8})\b"),
+    # The ACCOUNT part only, matching analytics.py. UA-1234-1 and UA-1234-2 are
+    # two properties of one account; keeping the suffix split them.
+    "ua": re.compile(r"\b(UA-\d{4,10})-\d{1,4}\b"),
+    "gtm": re.compile(r"\b(GTM-[A-Z0-9]{5,9})\b"),
     "fbpixel": re.compile(r"fbq\(['\"]init['\"],\s*['\"](\d{15,16})['\"]"),
     "hotjar": re.compile(r"hjid[:=]\s*(\d{6,8})"),
     "intercom": re.compile(r"app_id[\"']?\s*[:=]\s*[\"']([a-z0-9]{8})[\"']"),

{paytrace-2.0.2 → paytrace-2.0.4}/src/paytrace/net.py

@@ -174,6 +174,8 @@ class Fetcher:
         # of the investigation wrote.
         self.url_policy = url_policy or UrlPolicy()
         self.blocked: list[tuple[str, str]] = []
+        #: byte count and digest of the last scan-mode retrieval
+        self.last_scan: dict | None = None
         # Content varies by the requester's apparent location, so the exit used
         # is part of the evidence rather than a transport detail. A capture
         # that does not record its egress is not reproducible even in
@@ -281,7 +283,19 @@ class Fetcher:
         headers: dict | None = None,
         allow_html: bool = False,
         egress: str = "",
+        scanner=None,
+        scan_max_bytes: int | None = None,
     ) -> Response | None:
+        """Fetch a URL under the full policy, SSRF and pinning guards.
+
+        ``scanner`` switches to SCAN MODE: each decoded chunk is handed to the
+        callable and the body is NOT retained. That is what makes a resource
+        too large to hold usable -- Google's sellers.json is 104 MB, and
+        buffering it would put that in memory, in the cache, and in every
+        evidence package. In scan mode the evidence records the byte count and
+        a SHA-256 computed over the stream, so the retrieval stays verifiable
+        without storing it.
+        """
         # Policy is evaluated BEFORE the cache is consulted. Serving a cached
         # body skipped the check entirely, so `robots_policy: respect` stopped
         # applying the moment a URL had been fetched once -- including by an
@@ -313,7 +327,12 @@ class Fetcher:
                     return None
 
         cp = self._cache_path(url, headers, egress)
-        if cp.exists():
+        # Scan mode must not be served from the cache. The cache holds whatever
+        # the capped read produced -- for an oversized body that is a TRUNCATED
+        # copy -- and a scan needs the live stream. Serving the cache here made
+        # the streaming lookup silently find nothing whenever the URL had been
+        # fetched once already.
+        if cp.exists() and scanner is None:
             d = json.loads(cp.read_text())
             # A cache hit still has to appear in this run's evidence package.
             # Otherwise a second run consumed retrieved content and produced a
@@ -375,7 +394,9 @@ class Fetcher:
                                           outcome="refused", note=note)
                     return None
 
-        result = await self._get_following_redirects(url, headers, egress=egress)
+        result = await self._get_following_redirects(
+            url, headers, egress=egress,
+            scanner=scanner, scan_max_bytes=scan_max_bytes)
         if result is None:
             self._record_evidence(url, None, b"", egress, outcome="error",
                                   note="no response")
@@ -472,7 +493,8 @@ class Fetcher:
                 "the evidence package misleading."
             ) from e
 
-    async def _read_capped(self, response) -> tuple[bytes, bool]:
+    async def _read_capped(self, response, scanner=None,
+                           cap: int | None = None) -> tuple[bytes, bool]:
         """Read a response body, stopping at ``max_bytes``.
 
         ``response.content`` materialises the entire body first, so a hostile
@@ -483,21 +505,44 @@ class Fetcher:
         The cap applies to bytes *after* transfer decoding, so a decompression
         bomb is bounded by the same limit.
         """
-        buf = bytearray()
+        cap = cap or self.max_bytes
+        if scanner is None:
+            buf = bytearray()
+            truncated = False
+            async for chunk in response.aiter_bytes():
+                remaining = self.max_bytes - len(buf)
+                if remaining <= 0:
+                    truncated = True
+                    break
+                buf.extend(chunk[:remaining])
+                if len(chunk) > remaining:
+                    truncated = True
+                    break
+            return bytes(buf), truncated
+
+        # Scan mode: hand each chunk to the scanner and retain nothing. Memory
+        # stays flat regardless of body size, which is the whole point.
+        import hashlib
+
+        digest = hashlib.sha256()
+        seen = 0
         truncated = False
         async for chunk in response.aiter_bytes():
-            remaining = self.max_bytes - len(buf)
-            if remaining <= 0:
-                truncated = True
-                break
-            buf.extend(chunk[:remaining])
-            if len(chunk) > remaining:
+            if seen + len(chunk) > cap:
+                chunk = chunk[: cap - seen]
                 truncated = True
+            digest.update(chunk)
+            seen += len(chunk)
+            scanner(chunk)
+            if truncated:
                 break
-        return bytes(buf), truncated
+        self.last_scan = {"bytes": seen, "sha256": digest.hexdigest(),
+                          "truncated": truncated}
+        return b"", truncated
 
     async def _get_following_redirects(self, url: str, headers: dict | None,
-                                       max_hops: int = 5, egress: str = ""):
+                                       max_hops: int = 5, egress: str = "",
+                                       scanner=None, scan_max_bytes: int | None = None):
         """Follow redirects manually, validating every hop.
 
         Returns ``(status, body, encoding, truncated)``.
@@ -526,7 +571,8 @@ class Fetcher:
             try:
                 async with client.stream("GET", url, headers=headers) as r:
                     if r.status_code not in (301, 302, 303, 307, 308):
-                        body, truncated = await self._read_capped(r)
+                        body, truncated = await self._read_capped(
+                            r, scanner=scanner, cap=scan_max_bytes)
                         return (r.status_code, body, r.encoding, truncated, url,
                                 safe_headers(r.request.headers),
                                 safe_headers(r.headers))

{paytrace-2.0.2 → paytrace-2.0.4}/src/paytrace/sellersjson.py

@@ -248,12 +248,108 @@ def find_seller_in_text(body: str, adsystem: str, seller_id: str,
 
 
 async def resolve_seller(fetcher, adsystem: str, seller_id: str) -> SellerRecord | None:
-    """Fetch and locate a seller, trying every known location for the ad system."""
+    """Fetch and locate a seller, trying every known location for the ad system.
+
+    Falls back to a streaming scan when the document is too large to hold.
+    Google's sellers.json is 104 MB, so the ordinary body cap truncated it and
+    the payee could never be named -- for the ad system that serves the
+    majority of ad-funded sites.
+    """
     for url in candidate_urls(adsystem):
+        seen = len(getattr(fetcher, "blocked", ()))
         r = await fetcher.get(url)
-        if not r or r.status != 200 or not r.text:
+        truncated = any("truncated" in why
+                        for _u, why in list(getattr(fetcher, "blocked", ()))[seen:])
+        if r and r.status == 200 and r.text:
+            rec = find_seller_in_text(r.text, adsystem, seller_id, url)
+            if rec:
+                return rec
+        if truncated:
+            rec = await _stream_at(fetcher, url, adsystem, seller_id)
+            if rec:
+                return rec
+    return None
+
+
+# --------------------------------------------------------------------------- #
+# Streaming lookup, for sellers.json documents too large to hold
+# --------------------------------------------------------------------------- #
+
+#: Ceiling for a streamed sellers.json. Google's is 104 MB uncompressed, so the
+#: ordinary 10 MB body cap truncated it and the payee could never be named.
+#: Nothing is retained in scan mode, so this bounds time, not memory.
+STREAM_MAX_BYTES = 512 * 1024 * 1024
+
+#: Bytes kept between chunks so a record split across a boundary is still found.
+#: sellers.json records are a few hundred bytes; this is generous.
+_WINDOW = 64 * 1024
+
+
+class _RecordScanner:
+    """Find one seller record in a stream, without holding the stream."""
+
+    def __init__(self, seller_id: str) -> None:
+        self._needle = f'"{seller_id}"'.encode()
+        self._buf = b""
+        self.record: str | None = None
+
+    def feed(self, chunk: bytes) -> None:
+        if self.record is not None:
+            return
+        self._buf += chunk
+        i = self._buf.find(self._needle)
+        if i != -1:
+            found = _enclosing_object(self._buf, i)
+            if found is not None:
+                self.record = found
+                self._buf = b""
+                return
+        if len(self._buf) > _WINDOW:
+            self._buf = self._buf[-_WINDOW:]
+
+
+def _enclosing_object(buf: bytes, pos: int) -> str | None:
+    """The JSON object containing ``pos``, or None if it is not complete yet."""
+    start = buf.rfind(b"{", 0, pos)
+    if start == -1:
+        return None
+    depth, in_str, esc = 0, False, False
+    for j in range(start, len(buf)):
+        c = buf[j : j + 1]
+        if esc:
+            esc = False
             continue
-        rec = find_seller_in_text(r.text, adsystem, seller_id, url)
-        if rec:
-            return rec
+        if c == b"\\":
+            esc = True
+            continue
+        if c == b'"':
+            in_str = not in_str
+            continue
+        if in_str:
+            continue
+        if c == b"{":
+            depth += 1
+        elif c == b"}":
+            depth -= 1
+            if depth == 0:
+                return buf[start : j + 1].decode("utf-8", errors="replace")
     return None
+
+
+async def _stream_at(fetcher, url: str, adsystem: str, seller_id: str):
+    scanner = _RecordScanner(seller_id)
+    await fetcher.get(url, scanner=scanner.feed, scan_max_bytes=STREAM_MAX_BYTES)
+    if scanner.record is None:
+        return None
+    return find_seller_in_text('{"sellers":[' + scanner.record + "]}",
+                               adsystem, seller_id, url)
+
+
+async def stream_find_seller(fetcher, adsystem: str, seller_id: str):
+    """Look up one seller in a sellers.json of any size.
+
+    Streams the document, matches the record as it passes, and keeps only that
+    record. The retrieval stays evidenced: the fetcher records the byte count
+    and a SHA-256 taken over the stream.
+    """
+    return await _stream_at(fetcher, sellers_json_url(adsystem), adsystem, seller_id)

{paytrace-2.0.2 → paytrace-2.0.4}/tests/test_artifacts.py

@@ -200,3 +200,42 @@ async def test_staging_subdomains_become_pivots():
     claims = list(await c.collect(Identifier(IdKind.DOMAIN, "x.example")))
     subs = [x.object.value for x in claims if x.object.kind is IdKind.DOMAIN]
     assert "staging.realsite.example" in subs
+
+
+# ---- one identifier, one node --------------------------------------------- #
+
+def test_collectors_agree_on_the_canonical_identifier():
+    """`ca-pub-N` in page source and `pub-N` in ads.txt are one payee account.
+
+    Two collectors captured it in different shapes — digits in one, `pub-`
+    included in the other — so a single account became two graph nodes. Its
+    evidence was split between them, and on a real domain the same ID came out
+    STRONG_EVIDENCE under one spelling and UNSUPPORTED under the other.
+    """
+    from paytrace.collectors.analytics import ID_PATTERNS
+    from paytrace.collectors.artifacts import _SERVICE_IDS
+
+    digits = "5446113378742009"
+    for text in (f'src="//x/adsbygoogle.js?client=ca-pub-{digits}"',
+                 f"google.com, pub-{digits}, DIRECT"):
+        a = ID_PATTERNS["adsense"][0].search(text)
+        b = _SERVICE_IDS["adsense"].search(text)
+        assert a and b and a.group(1) == b.group(1) == digits, text
+
+
+def test_ua_identifiers_use_the_account_not_the_property():
+    """UA-1234-1 and UA-1234-2 are two properties of one account. Keeping the
+    property suffix in one collector and not the other split the account."""
+    from paytrace.collectors.analytics import ID_PATTERNS
+    from paytrace.collectors.artifacts import _SERVICE_IDS
+
+    for pattern in (ID_PATTERNS["ua"][0], _SERVICE_IDS["ua"]):
+        assert pattern.search("UA-12345678-3").group(1) == "UA-12345678"
+
+
+def test_no_second_scheme_name_for_one_service():
+    """`adsense_pub` was a separate scheme for the same accounts, which splits
+    a payee by scheme name rather than by value."""
+    from paytrace.collectors.analytics import ID_PATTERNS
+
+    assert "adsense_pub" not in ID_PATTERNS

{paytrace-2.0.2 → paytrace-2.0.4}/tests/test_sellersjson.py

@@ -165,3 +165,89 @@ def test_ext_field_survives_as_untrusted_extra():
     rec = find_seller_in_text(body, "x.example", "1")
     assert "ext" in rec.extra
     assert "terminate" in rec.extra["ext"]
+
+
+# ---- sellers.json documents too large to hold ------------------------------ #
+
+def test_scanner_finds_a_record_split_across_chunks():
+    """Google's sellers.json is 104 MB; it arrives in chunks and a record can
+    straddle any boundary."""
+    import json as _json
+
+    from paytrace.sellersjson import _RecordScanner
+
+    target = _json.dumps({"seller_id": "156423", "name": "Example Media Holdings Ltd"})
+    doc = ('{"sellers":['
+           + ",".join(_json.dumps({"seller_id": str(i), "name": f"Other {i}"})
+                      for i in range(200))
+           + "," + target + "]}").encode()
+    for size in (1, 7, 512, 65536):
+        scanner = _RecordScanner("156423")
+        for i in range(0, len(doc), size):
+            scanner.feed(doc[i:i + size])
+        assert scanner.record, f"missed at chunk size {size}"
+        assert _json.loads(scanner.record)["name"] == "Example Media Holdings Ltd"
+
+
+def test_scanner_reports_nothing_for_an_absent_seller():
+    """It must not return a neighbouring record."""
+    import json as _json
+
+    from paytrace.sellersjson import _RecordScanner
+
+    doc = ('{"sellers":[' + _json.dumps({"seller_id": "1", "name": "A"}) + "]}").encode()
+    scanner = _RecordScanner("999")
+    scanner.feed(doc)
+    assert scanner.record is None
+
+
+def test_oversized_sellers_json_is_streamed_and_resolved(monkeypatch):
+    """The ordinary 10 MB cap truncates Google's sellers.json, so the payee
+    could never be named — for the ad system serving most ad-funded sites.
+    Nothing is retained: the body is scanned as it arrives."""
+    import asyncio
+    import json as _json
+
+    import httpx
+
+    import paytrace.net as net
+    from paytrace.sellersjson import resolve_seller
+
+    target = _json.dumps({"seller_id": "pub-5446113378742009",
+                          "name": "Example Media Holdings Ltd",
+                          "domain": "x.example", "seller_type": "PUBLISHER"})
+    filler = ",".join(_json.dumps({"seller_id": str(i), "name": f"F{i}" * 40})
+                      for i in range(60000))
+    doc = ('{"sellers":[' + filler + "," + target + "]}").encode()
+    assert len(doc) > 10 * 1024 * 1024, "the body must exceed the cap to be a test"
+
+    def handler(req):
+        if req.url.path == "/robots.txt":
+            return httpx.Response(200, text="User-agent: *\nAllow: /\n")
+        if req.url.path.endswith("sellers.json"):
+            return httpx.Response(200, content=doc,
+                                  headers={"content-type": "application/json"})
+        return httpx.Response(404)
+
+    original = net.Fetcher.__init__
+
+    def patched(self, *a, **k):
+        k["resolver"] = lambda host: ["93.184.216.34"]
+        original(self, *a, **k)
+        for route in self.egress.egresses:
+            self._clients[route.label] = httpx.AsyncClient(
+                transport=httpx.MockTransport(handler), follow_redirects=False)
+
+    monkeypatch.setattr(net.Fetcher, "__init__", patched)
+
+    async def go():
+        f = net.Fetcher(user_agent="t")
+        rec = await resolve_seller(f, "google.com", "pub-5446113378742009")
+        scan = f.last_scan
+        await f.aclose()
+        return rec, scan
+
+    rec, scan = asyncio.run(go())
+    assert rec and rec.name == "Example Media Holdings Ltd"
+    assert scan and scan["bytes"] == len(doc), "the whole body must be streamed"
+    assert scan["sha256"], "the retrieval must stay evidenced by a digest"