rsc-universal 0.1.1

package/skills/data-policy/scripts/verify.sh

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+# verify.sh — data-policy
+# Read-only, offline, deterministic lint of a GENERATED governance artifact
+# (a finished retention schedule / ROPA / consent matrix / data policy).
+# Exits 0 on a clean OR empty/missing target (no false failure).
+# Non-zero only on a real violation. grep/awk only, no deps. bash 3.2-safe.
+#
+# Checks (per the plan):
+#   1. A retention/schedule table row must not be missing its period or lawful
+#      basis (empty or dash cell under a period/retention/basis column).
+#   2. Vague period language ("as long as necessary", "indefinitely", "forever",
+#      "permanently", "until no longer needed") must NOT be the only period —
+#      fail if present and no concrete period (number+unit, or "criteria") exists.
+#   3. A full policy must carry a disclaimer: "DPO" / "counsel" /
+#      "legal review" / "not legal advice".
+#   4. If the text mentions deletion it must also mention "backup" or "archive"
+#      (the enforcement gap).
+#
+# This lints FINISHED artifacts only. Blank-celled templates (a file whose
+# heading or text marks it a "template", or that carries empty `| |` fill-in
+# rows) are skipped — an unfilled template is not a failed policy.
+#
+# Usage: verify.sh [FILE_OR_DIR]   (default: current directory)
+#        verify.sh -               (read from stdin)
+
+set -uo pipefail
+
+TARGET="${1:-.}"
+
+TMP=""
+cleanup() { [ -n "$TMP" ] && rm -f "$TMP"; }
+trap cleanup EXIT
+
+# --- gather candidate files (or stdin), bash-3.2 safe ------------------------
+LIST="$(mktemp)"
+trap 'cleanup; rm -f "$LIST"' EXIT
+
+if [ "$TARGET" = "-" ]; then
+  TMP="$(mktemp)"
+  cat > "$TMP"
+  printf '%s\n' "$TMP" > "$LIST"
+elif [ -d "$TARGET" ]; then
+  find "$TARGET" -type f \( -name '*.md' -o -name '*.markdown' -o -name '*.txt' \) 2>/dev/null > "$LIST"
+elif [ -f "$TARGET" ]; then
+  printf '%s\n' "$TARGET" > "$LIST"
+else
+  echo "verify: target '$TARGET' not found; nothing to check."
+  exit 0
+fi
+
+if [ ! -s "$LIST" ]; then
+  echo "verify: no candidate files under '$TARGET'; clean by default."
+  exit 0
+fi
+
+fail=0
+checked=0
+note_fail() { echo "FAIL: $1"; fail=1; }
+
+VAGUE='as long as necessary|indefinitely|forever|permanently|until no longer needed'
+CONCRETE='[0-9]+[[:space:]]*(day|days|week|weeks|month|months|year|years|yr|yrs|mo)|criteria'
+DISCLAIMER='DPO|counsel|legal review|not legal advice'
+DELETION='delet|erase|purge'
+BACKUPS='backup|archive'
+ARTIFACT='retention|lawful basis|record of processing|ropa|consent matrix|storage limitation|expiry'
+
+while IFS= read -r f; do
+  [ -n "$f" ] || continue
+  [ -f "$f" ] || continue
+  label="$f"; [ "$f" = "$TMP" ] && label="<stdin>"
+
+  # Only lint files that look like a governance ARTIFACT, not prose docs that
+  # merely name retention. Require an artifact signal AND a markdown table
+  # (a line with two or more pipes) — schedules/ROPAs/consent matrices are tabular.
+  grep -qiE "$ARTIFACT" "$f" 2>/dev/null || continue
+  grep -qE '\|.*\|' "$f" 2>/dev/null || continue
+
+  # Skip unfilled TEMPLATES: a file that announces itself a template, or carries
+  # an empty markdown fill-in row like "|  |  |  |". Those legitimately have
+  # blank cells and are not a failed policy.
+  if grep -qiE 'template|fill-?able|fill-?in|copy-?ready' "$f" 2>/dev/null \
+     || grep -qE '^\|[[:space:]]*\|([[:space:]]*\|)+[[:space:]]*$' "$f" 2>/dev/null; then
+    continue
+  fi
+
+  checked=$((checked+1))
+
+  # --- Check 1: table rows missing period or lawful basis --------------------
+  awk -v FNAME="$label" '
+    function trim(s){ gsub(/^[ \t]+|[ \t]+$/,"",s); return s }
+    BEGIN{ FS="|"; rc=0 }
+    /\|/ && /[Pp]eriod|[Rr]etention|[Ll]awful basis|[Bb]asis/ {
+      delete col
+      for (i=1;i<=NF;i++){
+        c=tolower(trim($i))
+        if (c ~ /period|retention/ || c ~ /lawful basis|^basis$/){ col[i]=c }
+      }
+      n=0; for (k in col) n++
+      if (n>0){ inhdr=NR; next }
+    }
+    # an optional markdown separator row (---|---) right after the header
+    inhdr && NR==inhdr+1 && /^[ \t]*\|?[ \t:|-]+$/ { next }
+    inhdr && NR>inhdr {
+      if ($0 !~ /\|/){ inhdr=0; next }
+      for (i in col){
+        cell=trim($i)
+        if (cell=="" || cell ~ /^-+$/){
+          printf("FAIL: %s: table row %d missing %s cell\n", FNAME, NR, col[i]); rc=1
+        }
+      }
+    }
+    END{ exit rc }
+  ' "$f" || fail=1
+
+  # --- Check 2: vague period as the ONLY period ------------------------------
+  if grep -qiE "$VAGUE" "$f" 2>/dev/null; then
+    grep -qiE "$CONCRETE" "$f" 2>/dev/null \
+      || note_fail "$label: vague period with no concrete period or criteria"
+  fi
+
+  # --- Check 3: missing disclaimer -------------------------------------------
+  grep -qiE "$DISCLAIMER" "$f" 2>/dev/null \
+    || note_fail "$label: no DPO / counsel / legal-review / 'not legal advice' disclaimer"
+
+  # --- Check 4: deletion without backups/archives ----------------------------
+  if grep -qiE "$DELETION" "$f" 2>/dev/null; then
+    grep -qiE "$BACKUPS" "$f" 2>/dev/null \
+      || note_fail "$label: mentions deletion but never backups/archives (the enforcement gap)"
+  fi
+done < "$LIST"
+
+if [ "$fail" -ne 0 ]; then
+  echo "verify: violations found."
+  exit 1
+fi
+
+if [ "$checked" -eq 0 ]; then
+  echo "verify: no finished policy artifacts to lint (templates/non-artifacts only); clean by default."
+  exit 0
+fi
+
+echo "verify: OK — periods + bases present, no vague-only periods, disclaimer present, backups covered."
+exit 0

package/skills/data-scraper/SKILL.md

@@ -0,0 +1,134 @@
+---
+name: data-scraper
+description: "Use when you need data that lives on a website with no usable API — listings, prices, public records, directories — and the scrape must stay legal and not get blocked. Triggers: 'scrape this site', 'extract data from a webpage', 'build a crawler', 'pull all the prices off this catalog', 'my scraper keeps getting 429s / IP-banned', 'the selectors break every time the site redesigns', 'is it legal to scrape this without getting sued', 'extreure dades d'una web', 'scrapear una página sin que me bloqueen'. NOT parsing HTML/PDF you already have into fields (that is structured-extraction), NOT a site that already exposes a documented API or key (that is api-connector-builder)."
+tags: [web-scraping, crawler, anti-bot, robots-txt, gdpr-compliance, playwright, rate-limiting]
+recommends: [api-connector-builder, structured-extraction, data-cleaning, gdpr-privacy, webhooks]
+origin: risco
+---
+
+# Data scraper
+
+You get bytes off websites you do not control — legally, and without getting blocked. You pick the cheapest extraction path that works, build selectors that survive a redesign, pace requests so the host neither bans nor sues you, and you write down the legal basis *before* the first request goes out.
+
+One rule above all the others: **scraping is the fallback, not the default.** It is what you reach for only when no API serves the data. If the site has a documented API or you hold a key, stop — that is `../api-connector-builder/SKILL.md`. And once you have the bytes, parsing them into fields is `../structured-extraction/SKILL.md`, normalizing the rows is `../data-cleaning/SKILL.md`. This skill ends the moment you hold the bytes.
+
+In 2025-2026 scrapers do not fail on parsing. They fail on a terms-of-service breach, on GDPR exposure, or on being blocked after hammering a host. So the work runs in this order: legal gate → extraction path → tool → selectors → politeness → resilience. Skipping the gate is how you end up in *Meta v. Bright Data*.
+
+## The legal gate — run this before any request
+
+Walk every item. Each ends in **proceed**, **proceed-narrowed**, or **stop**. One item at stop means the whole scrape stops until you resolve it. Depth and the case law are in `references/legal-compliance.md`.
+
+- [ ] **Is there an API?** If yes, you are in the wrong skill — never scrape what an API serves. → otherwise proceed.
+- [ ] **Did you read the ToS, and does login/auth apply?** Scraping is most exposed as *breach of contract* when you accepted terms — typically by logging in (*Meta v. Bright Data*, 2024). Logged-out public data weakens that claim. **Prefer logged-out public pages; never bypass auth.** → narrow to public, or stop.
+- [ ] **Is it personal data?** Names, emails, photos, reviews, IP addresses all count. Scraping public personal data for a *new* purpose — aggregation, resale, AI training — is a severe GDPR breach with fines into the tens of millions of EUR. You need a lawful basis (usually legitimate interest) and data minimization. Filter out special categories at the source. → proceed-narrowed (basis + minimization, see `../gdpr-privacy/SKILL.md`), or stop.
+- [ ] **robots.txt and ai.txt?** Honor disallow rules and AI-objection signals. Ignoring them is the first fact cited against you. → narrow the path set, or stop.
+- [ ] **Are you about to bypass a control you were shown?** A CAPTCHA, a hard block, an enforced rate limit. *Reddit v. Perplexity AI* (2025) turns precisely on whether anti-bot measures were circumvented — a materially worse position than respectfully pacing public pages. Pacing public data is defensible; defeating a control is the frontier where you lose. → **stop. Do not solve the CAPTCHA. Do not bypass the block.**
+
+`hiQ v. LinkedIn` established that scraping *public* data is not automatically a CFAA violation — but that is the floor, not a license. Public + logged-out + robots-honored + no-personal-data is the defensible quadrant. Anything else, document why and get a human to sign off.
+
+## Extraction-path decision table
+
+Walk *down* only when the rung above is unavailable. ~94% of modern sites are client-side rendered, yet most still ship machine-readable structured data — parse that before you launch a browser.
+
+| Path | Use when | Cost | Detectability |
+|------|----------|------|---------------|
+| **API** (incl. internal XHR/JSON endpoints) | Any documented API, or the page fetches its own JSON you can call directly | Lowest | Lowest — looks like normal traffic |
+| **sitemap.xml + JSON-LD** | `/sitemap.xml` lists the URLs; `<script type="application/ld+json">` carries the records (JSON-LD is Google's preferred structured format, so it is everywhere) | Low — one HTTP GET, parse JSON | Low |
+| **HTML selectors** | Data is in server-rendered HTML, no JSON-LD, no XHR JSON | Medium — selectors drift on redesign | Medium |
+| **Headless browser** | The data only exists after JS executes (SPA, lazy-load, infinite scroll) and there is no callable XHR endpoint | Highest — CPU, RAM, time, easiest to fingerprint | Highest |
+
+Before you reach for a browser, open DevTools → Network and look for the XHR/fetch that already returns JSON. Calling that endpoint directly is faster, stabler, and less detectable than rendering the whole page to read what the page itself fetched.
+
+## Tool picker
+
+| Site profile | Tool | Version (2026) | The one reason |
+|--------------|------|----------------|----------------|
+| Static HTML, no fingerprint wall | `httpx` + `selectolax` (or BeautifulSoup) | current | Fast, no browser; `selectolax` parses far quicker than `lxml` |
+| Static HTML but TLS/JA3 fingerprint blocks you | `curl_cffi` (profile `chrome131`) | current | Impersonates a real browser's TLS/JA3/HTTP2 fingerprint, not just the User-Agent |
+| JS-rendered SPA | Playwright | 1.60.0 (1.59 shipped 2026-04-01) | First-class async, auto-wait, the maintained headless standard |
+| Scalable, resilient, recurring crawler | Crawlee (JS or Python) | actively maintained 2026 | Wraps Playwright + proxy rotation + browserforge fingerprints + a disk-persisted `RequestQueue` that resumes after a crash |
+| Pure-Python static, legacy codebase | Scrapy | current | Battle-tested for static targets — but its Twisted core lags the asyncio ecosystem; pick Crawlee for new work |
+
+Default new builds to **Crawlee** when the job is recurring or must not break; reach for plain `httpx`/`curl_cffi` only when the target is static and one-shot.
+
+## Robust selectors
+
+Selectors break on redesign because they ride on layout, not meaning. Anchor on what is *semantically* stable — `data-*` attributes, ARIA roles, microdata, visible text — never on `nth-child` chains or generated CSS class hashes (`.css-1a2b3c`), which change on every build.
+
+```html
+<!-- the page you are scraping -->
+<article data-testid="listing-card">
+  <h2 class="css-1a2b3c">Acme Drill 9000</h2>
+  <span data-price="129.00">€129,00</span>
+</article>
+```
+
+```python
+# Bad — rides on layout and a build-generated hash; dies on the next deploy
+title = page.query_selector("div:nth-child(3) > article > .css-1a2b3c").inner_text()
+
+# Good — anchor on stable semantics, with a fallback chain, and fail loud
+def text_or_raise(card, selectors, field):
+    for sel in selectors:           # try each selector in priority order
+        el = card.query_selector(sel)
+        if el and el.inner_text().strip():
+            return el.inner_text().strip()
+    raise LookupError(f"required field {field!r} not found via {selectors}")
+
+card  = page.query_selector('[data-testid="listing-card"]')
+title = text_or_raise(card, ["h2", '[itemprop="name"]'], "title")
+price = text_or_raise(card, ["[data-price]", "span:has-text('€')"], "price")
+```
+
+Two rules baked into that snippet:
+
+- **Fallback selector chains.** Give every required field 2-3 ordered selectors. A single redesign rarely breaks all of them at once, so the scraper degrades instead of dying.
+- **Fail loud on a missing required field.** Raise — never silently write `null`. A silent null is a corrupted dataset you discover three months later. A raised error is a fix you make today.
+
+## Politeness and anti-block
+
+Concrete numbers beat "be respectful." Depth — fingerprint profiles, header sets, proxy taxonomy — is in `references/anti-bot.md`.
+
+- **Pace.** Start at **1 request per 1-3 seconds per host**, concurrency capped at **2-5 per host**. Tune *down* if you see 429s, never up to chase speed.
+- **Backoff with jitter.** On a transient error, exponential backoff *plus* random jitter — `delay = base * 2**attempt + random(0, base)`. Without jitter, every worker retries in lockstep and you self-DDoS the host.
+- **Honor 429 and `Retry-After`.** A 429 with `Retry-After: 30` means wait 30 seconds, not retry immediately. Ignoring it is the fastest route to an IP ban.
+- **Conditional GET.** Send `If-Modified-Since` / `If-None-Match` (ETag) on re-crawls. A `304 Not Modified` costs nothing and tells you the page is unchanged — cheaper for you, lighter on the host.
+- **Realistic headers + TLS fingerprint.** A bare `python-requests` User-Agent is an instant tell. Send a full, current browser header set; when JA3/TLS fingerprinting blocks you, switch to `curl_cffi` (`chrome131`) — see the references.
+- **Proxies only when justified.** Residential/mobile proxies are for geo-gating and IP-rate distribution on a target that *permits* the scrape — never to evade a block you were explicitly shown. Budget for them on large recurring jobs; do not reach for them to defeat a hard block (that is the legal line you do not cross).
+
+> **Resilience beats speed.** A scraper that runs 50% slower but never breaks is infinitely more valuable than a fast one that dies weekly. Pace for survival, not throughput.
+
+## Resilience
+
+A recurring crawler must survive crashes, redeploys, and the target's schema drift. Patterns and copy-paste starters are in `references/frameworks.md`.
+
+- **Resumable queue.** Use a disk-persisted request queue (Crawlee's `RequestQueue`) so a crash resumes from where it stopped, not from zero. Re-crawling 10k pages because the box rebooted is wasted budget and extra load on the host.
+- **Idempotent writes.** Upsert keyed on a stable id (the source URL or a record id), never blind append — so a re-run repairs rows instead of duplicating them.
+- **Checkpoint every N records.** Flush progress periodically so an interrupt loses minutes, not hours.
+- **Change detection by hash.** Store a content hash per record; only re-process when it changes. This pairs with conditional GET to keep recurring crawls cheap.
+- **Monitor and alert.** Alert on a spike in missing-field errors (schema drift — the site redesigned) and on a spike in 403/429 (you are being blocked). Both are silent-failure modes that rot a dataset until someone checks.
+
+## Anti-patterns
+
+| Anti-pattern | Why it bites | Do instead |
+|---|---|---|
+| Scraping behind a login, then calling it "public data" | Accepting ToS at login is the breach-of-contract hook (*Meta v. Bright Data*) | Stay logged-out on public pages; never bypass auth |
+| Ignoring robots.txt / ai.txt | First fact cited against you; signals bad faith | Parse and honor both before queuing URLs |
+| No delay, unbounded concurrency | Hammers the host → IP ban, possible CFAA-style exposure | 1 req / 1-3s, cap 2-5 concurrent per host |
+| Selectors on `nth-child` / `.css-1a2b3c` hashes | Break on the next deploy; silent data loss | Anchor on `data-*` / semantic / text, with fallbacks |
+| Silently writing `null` on a missing field | Corrupts the dataset; discovered months later | Raise on a missing *required* field — fail loud |
+| Solving a CAPTCHA / bypassing a hard block | Circumventing a shown control (*Reddit v. Perplexity*) — the worst legal posture | Stop. That control is a "no." |
+| Scraping personal data with no lawful basis | GDPR fines into tens of millions EUR | Establish basis + minimize + filter special categories (`../gdpr-privacy/SKILL.md`) |
+| Storing everything "just in case" | Defeats minimization; expands breach blast radius | Keep only fields the purpose needs; set retention |
+| Launching a browser when JSON-LD was right there | Slowest, most detectable, most expensive path | Check XHR/JSON-LD/sitemap first; browser is last |
+| No resume — a crash restarts from zero | Wastes budget, doubles load on the host | Disk-persisted resumable queue |
+| Hardcoding one User-Agent forever | Stale UA is an easy bot tell | Current full header set; rotate when justified |
+| Retrying with no backoff | Lockstep retries self-DDoS the host | Exponential backoff + jitter; honor `Retry-After` |
+
+## References
+
+- **`references/legal-compliance.md`** — robots.txt + ai.txt parsing, ToS red-flag list, GDPR lawful-basis decision flow, minimization/retention checklist, and the 2024-2025 case summaries (Meta v. Bright Data, hiQ v. LinkedIn, Reddit v. Perplexity).
+- **`references/anti-bot.md`** — `curl_cffi` impersonation profiles with an example, realistic header sets, Playwright launch/stealth config, the proxy taxonomy (datacenter / residential / mobile) and when each is justified, and the CAPTCHA line you do not cross.
+- **`references/frameworks.md`** — copy-paste resilient starters: Crawlee Python (`RequestQueue` + router + per-host concurrency), Playwright direct, Scrapy AUTOTHROTTLE, and an `httpx` + `selectolax` static template.
+
+When in doubt about whether a scrape is defensible, the answer is the gate. Run it, write down the outcome, and only then send a request.

package/skills/data-scraper/evals/README.md

@@ -0,0 +1,3 @@
+# Evals: data-scraper
+
+These cases are routing and coverage checks, not an automated pass/fail suite. Read `cases.yaml` against `SKILL.md`. For each `should_trigger` prompt, confirm the description and body would plausibly fire this skill rather than a sibling — note the two non-obvious symptom cases (429/IP-ban and selectors-break-on-redesign, which read as ops problems but are core here) and the Spanish and Catalan triggers. For each `should_not_trigger` prompt, confirm the named `route_to` sibling is the better home and that the boundary in the description would defer to it (structured-extraction once you hold bytes, api-connector-builder when an API exists, data-cleaning for normalization, e2e-testing for UI flow automation, gdpr-privacy for a pure lawful-basis question). For the `capability` case, have a human or a harness LLM produce the response and check it hits every `must_include` item — above all that it runs the legal gate first, walks the extraction path before reaching for a browser, paces per host with backoff and honors 429, uses a resumable queue, fails loud on missing fields, and refuses to bypass a CAPTCHA or hard block. There is no scripted assertion here. The separate `scripts/verify.sh` is the only automated check, and it statically lints an emitted scraper file for a compliance preflight, a rate limit/concurrency cap, retry/backoff, and the absence of CAPTCHA-solver / auth-bypass calls — not these cases.

package/skills/data-scraper/evals/cases.yaml

@@ -0,0 +1,46 @@
+skill: data-scraper
+
+should_trigger:
+  - prompt: "I need to pull all product prices off this competitor's catalog site — they have no API."
+    why: Core case — structured data off a site with no API, the canonical scrape.
+  - prompt: "Build me a crawler that grabs new public tender listings every morning."
+    why: Recurring resilient crawler — routes to the resilience + tool-picker sections.
+  - prompt: "My scraper keeps getting 429s and then the IP gets banned after about 200 pages."
+    why: Non-obvious — the symptom is a politeness/anti-block failure, not a parsing one.
+  - prompt: "The CSS selectors in my scraper break every time the site redesigns."
+    why: Non-obvious — points at the robust-selectors section (data-attr / semantic / fallback chains).
+  - prompt: "Is it legal for me to scrape this site, and how do I do it without getting sued?"
+    why: Legal-gate first — ToS, robots.txt, personal-data, circumvention before any request.
+  - prompt: "Vull extreure les dades de contacte d'un directori d'empreses públic."
+    why: Catalan trigger — public directory scrape that also nudges the GDPR personal-data gate.
+  - prompt: "Necesito scrapear una página de anuncios sin que me bloqueen."
+    why: Spanish trigger — extraction plus the anti-block concern.
+
+should_not_trigger:
+  - prompt: "Parse these 500 saved HTML invoices into a fields table."
+    route_to: structured-extraction
+    why: Bytes are already in hand; the task is field extraction from documents, not fetching.
+  - prompt: "The site publishes a documented REST API with a key — pull the orders into our DB."
+    route_to: api-connector-builder
+    why: A real API exists; never scrape what an API serves.
+  - prompt: "Dedup and normalize this messy scraped CSV — fix the types and nulls."
+    route_to: data-cleaning
+    why: Downstream of having the rows; normalization, not extraction.
+  - prompt: "Automate clicking through our checkout flow to test it end to end."
+    route_to: e2e-testing
+    why: Browser UI automation for testing a flow, not data extraction.
+  - prompt: "What's my GDPR lawful basis for processing this customer list we already hold?"
+    route_to: gdpr-privacy
+    why: Pure legal-basis question with no scrape — the privacy artifact layer.
+
+capability:
+  - scenario: "Stand up a resilient daily crawler for a public, JS-rendered listings site that has no API. It contains business names and contact emails. Tell me how to build it so it stays legal and doesn't get blocked."
+    must_include:
+      - Runs the legal gate before any request — reads ToS, checks robots.txt/ai.txt, and flags the contact emails as personal data needing a lawful basis + minimization (defers basis detail to gdpr-privacy)
+      - Walks the extraction path — checks for a callable XHR/JSON endpoint, sitemap.xml, and JSON-LD before committing to a headless browser
+      - Picks a tool with a reason — Playwright (1.60) or Crawlee for the JS-rendered SPA, not plain httpx
+      - Builds selectors on stable attributes (data-* / semantic / text) with a fallback chain, and fails loud on a missing required field rather than writing null
+      - Sets a per-host rate limit (≈1 req / 1-3s, concurrency cap 2-5), exponential backoff with jitter, and honors 429 / Retry-After
+      - Uses a resumable disk-persisted queue (Crawlee RequestQueue) and idempotent upserts keyed on a stable id so a crash resumes instead of restarting
+      - Refuses to bypass any CAPTCHA or hard block, treating it as a stop and re-running the gate
+      - Adds monitoring/alerts on missing-field spikes (schema drift) and 403/429 spikes (blocking)

package/skills/data-scraper/references/anti-bot.md

@@ -0,0 +1,85 @@
+# Anti-bot reference
+
+Anti-bot handling is mandatory in 2026, not optional — basic Puppeteer worked in 2024, but every major site now ships detection. This covers the *legitimate* techniques: realistic fingerprints and pacing on a scrape you are entitled to run. It does **not** cover defeating a control you were shown — that is the legal line from `legal-compliance.md`, and you do not cross it.
+
+## TLS / JA3 fingerprinting with curl_cffi
+
+Many blocks key on the TLS/JA3/HTTP2 fingerprint, not the User-Agent. A real Chrome handshake looks different from a Python `requests` handshake regardless of headers, so spoofing the UA alone fails. `curl_cffi` impersonates a real browser's full fingerprint.
+
+```python
+from curl_cffi import requests
+
+# impersonate="chrome131" sends Chrome's real TLS/JA3 + HTTP2 fingerprint
+r = requests.get(
+    "https://example.com/listings",
+    impersonate="chrome131",
+    timeout=20,
+)
+r.raise_for_status()
+html = r.text
+```
+
+Use this the moment a static target returns 403 to `httpx` but 200 in a real browser — that gap is almost always fingerprinting.
+
+## Realistic header set
+
+Send a full, current browser header set, not a lone User-Agent. A bare `python-requests/2.x` UA is an instant tell.
+
+```python
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                  "AppleWebKit/537.36 (KHTML, like Gecko) "
+                  "Chrome/131.0.0.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Upgrade-Insecure-Requests": "1",
+}
+```
+
+Keep the UA current — a stale Chrome version is its own tell. Refresh it when the major version moves.
+
+## Playwright launch config
+
+```python
+from playwright.async_api import async_playwright
+
+async def fetch(url: str) -> str:
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        ctx = await browser.new_context(
+            user_agent=HEADERS["User-Agent"],
+            locale="en-US",
+            viewport={"width": 1366, "height": 768},
+        )
+        page = await ctx.new_page()
+        await page.goto(url, wait_until="networkidle", timeout=30_000)
+        html = await page.content()
+        await browser.close()
+        return html
+```
+
+For scale, prefer Crawlee's `AdaptivePlaywrightCrawler` with browserforge fingerprints over hand-rolled stealth — it picks the cheapest mode (HTTP vs browser) per request and ships coherent fingerprints. See `frameworks.md`.
+
+## Proxy taxonomy — when each is justified
+
+Proxies distribute IP load and resolve geo-gating on a scrape you are entitled to run. They are not for evading a block you were explicitly shown.
+
+| Type | Cost | Justified when |
+|------|------|----------------|
+| Datacenter | Lowest | High-volume scrape of a permissive target; easiest to detect/block |
+| Residential | Medium | Geo-gated content; target blocks datacenter ranges but permits the scrape |
+| Mobile | Highest | Mobile-only content or the strictest IP reputation requirements |
+
+Rule: reach for a proxy to spread load or reach a geo, never to defeat a hard block. The latter is circumvention.
+
+## CAPTCHA — the line you do not cross
+
+A CAPTCHA is the site explicitly saying "prove you are human or do not proceed." Routing it to a solver service to keep scraping is circumventing a shown access control — exactly the posture at issue in *Reddit v. Perplexity AI* (2025), and the worst place to be legally.
+
+When you hit a CAPTCHA or a hard block: **stop.** Re-run the legal gate. Slow down, narrow the path set, or seek permission/an API. Do not solve it.
+
+Sources: brightdata.com "Web Scraping With curl_cffi 2026"; curl-cffi.readthedocs.io; dataresearchtools.com TLS mimicry 2026; crawlee.dev/python docs. All accessed 2026-06-02.

package/skills/data-scraper/references/frameworks.md

@@ -0,0 +1,116 @@
+# Frameworks reference
+
+Copy-paste resilient starters. Match the tool to the site profile from the picker in `../SKILL.md`. Every starter assumes the legal gate already passed.
+
+## Crawlee (Python) — resilient recurring crawler
+
+Default for recurring jobs that must not break. The `RequestQueue` is disk-persisted, so a crash resumes from where it stopped. Per-host concurrency and pacing are config, not hand-rolled.
+
+```python
+import asyncio
+from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
+
+crawler = PlaywrightCrawler(
+    max_requests_per_crawl=10_000,
+    max_request_retries=4,                 # retries use backoff internally
+    concurrency_settings={"max_concurrency": 4},  # cap per run; pace per host below
+    request_handler_timeout_secs=45,
+)
+
+@crawler.router.default_handler
+async def handle(ctx: PlaywrightCrawlingContext) -> None:
+    title = await ctx.page.text_content('[data-testid="listing-card"] h2')
+    if not title:
+        raise LookupError(f"required field 'title' missing at {ctx.request.url}")  # fail loud
+    await ctx.push_data({                  # idempotent: keyed on url downstream
+        "url": ctx.request.url,
+        "title": title.strip(),
+    })
+    await ctx.enqueue_links(selector='a[href^="/listing/"]')  # follow only listing links
+
+async def main() -> None:
+    # seed from sitemap, not a blind crawl
+    await crawler.run(["https://example.com/sitemap.xml"])
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+Why Crawlee for new resilient work: it wraps Playwright, rotates proxies, ships browserforge fingerprints, and the queue resumes after a crash — all the resilience patterns from `../SKILL.md` come built in instead of hand-rolled.
+
+## Playwright direct — one-off SPA scrape
+
+When the job is one-shot and a full crawler is overkill, but the data only exists after JS runs.
+
+```python
+import asyncio, random
+from playwright.async_api import async_playwright
+
+async def scrape(urls: list[str]) -> list[dict]:
+    out = []
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        ctx = await browser.new_context(locale="en-US")
+        for url in urls:
+            page = await ctx.new_page()
+            await page.goto(url, wait_until="networkidle", timeout=30_000)
+            title = await page.text_content('[data-testid="listing-card"] h2')
+            if not title:
+                raise LookupError(f"required field 'title' missing at {url}")
+            out.append({"url": url, "title": title.strip()})
+            await page.close()
+            await asyncio.sleep(random.uniform(1.0, 3.0))  # 1-3s per host, jittered
+        await browser.close()
+    return out
+```
+
+## Scrapy — pure-Python static, AUTOTHROTTLE
+
+Legacy/static targets. AUTOTHROTTLE adapts the delay to the host's response latency instead of a fixed sleep.
+
+```python
+# settings.py
+ROBOTSTXT_OBEY = True               # honor robots.txt, non-negotiable
+AUTOTHROTTLE_ENABLED = True
+AUTOTHROTTLE_START_DELAY = 1.0      # seconds
+AUTOTHROTTLE_MAX_DELAY = 10.0
+AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0   # keep it low, per host
+CONCURRENT_REQUESTS_PER_DOMAIN = 4
+DOWNLOAD_DELAY = 1.0
+RETRY_ENABLED = True
+RETRY_TIMES = 3                     # Scrapy backs off between retries
+DEFAULT_REQUEST_HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
+HTTPCACHE_ENABLED = True            # conditional-GET style caching on re-crawl
+```
+
+Pick Scrapy only for an existing Scrapy codebase or pure static targets — its Twisted core lags the asyncio ecosystem, so new resilient work goes to Crawlee.
+
+## httpx + selectolax — static, no fingerprint wall
+
+Cheapest path when the HTML is server-rendered and the host does not fingerprint.
+
+```python
+import time, random, httpx
+from selectolax.parser import HTMLParser
+
+def scrape(urls: list[str], headers: dict) -> list[dict]:
+    out = []
+    with httpx.Client(headers=headers, timeout=20, follow_redirects=True) as c:
+        for url in urls:
+            r = c.get(url)
+            if r.status_code == 429:                       # honor Retry-After
+                time.sleep(int(r.headers.get("Retry-After", "30")))
+                r = c.get(url)
+            r.raise_for_status()
+            tree = HTMLParser(r.text)
+            node = tree.css_first('[data-testid="listing-card"] h2')
+            if node is None:
+                raise LookupError(f"required field 'title' missing at {url}")
+            out.append({"url": url, "title": node.text(strip=True)})
+            time.sleep(random.uniform(1.0, 3.0))           # 1-3s per host
+    return out
+```
+
+If this starts returning 403 while a browser returns 200, swap `httpx` for `curl_cffi` with `impersonate="chrome131"` (see `anti-bot.md`).
+
+Sources: crawlee.dev/python changelog; npmjs.com/package/playwright & pypi.org/project/playwright (1.60.0; 1.59 shipped 2026-04-01); use-apify.com "Crawlee vs Scrapy vs BeautifulSoup 2026". All accessed 2026-06-02.

package/skills/data-scraper/references/legal-compliance.md

@@ -0,0 +1,59 @@
+# Legal & compliance reference
+
+Depth behind the legal gate in `../SKILL.md`. You are not a lawyer and you never claim to be — you document the basis and flag where a human must sign off.
+
+## robots.txt and ai.txt
+
+Fetch and honor both before queuing a single URL. `robots.txt` governs crawler access by user-agent and path; `ai.txt` (and the `noai` / `noimageai` signals some sites publish) is the emerging objection channel for AI training use specifically.
+
+```python
+import urllib.robotparser, urllib.parse
+
+def allowed(url: str, ua: str = "my-crawler") -> bool:
+    base = "{0.scheme}://{0.netloc}".format(urllib.parse.urlparse(url))
+    rp = urllib.robotparser.RobotFileParser()
+    rp.set_url(base + "/robots.txt")
+    rp.read()                      # network call; cache per host
+    return rp.can_fetch(ua, url)   # False -> do not queue this URL
+```
+
+Rules:
+
+- A `Disallow` for your path is a stop for that path, not a suggestion. Narrow the URL set.
+- Respect `Crawl-delay` if present — it is the host telling you its tolerance.
+- An `ai.txt` / `noai` objection means do not use the data for AI training, even if the page is public. Honor it; it is the cleanest evidence of good faith.
+
+## ToS red flags
+
+Read the terms before scraping. The presence of any of these moves you toward **narrow** or **stop**:
+
+- An explicit "no automated access / no scraping / no crawling" clause.
+- Acceptance gated by **login** — accepting at sign-in is the contract hook that made *Meta v. Bright Data* a breach-of-contract case rather than a CFAA one.
+- A clause assigning IP/database rights over the listings themselves (common on directories and aggregators).
+- An API offered under separate commercial terms — using it routes you to `../api-connector-builder/SKILL.md` and scraping around it looks like circumvention.
+
+## GDPR lawful-basis decision flow
+
+Personal data = names, emails, photos, reviews, IP addresses, anything identifying a natural person. If the scrape touches any:
+
+1. **New purpose check.** Is your purpose the same as why the data was published? Reusing public personal data for aggregation, resale, or AI training is a *new* purpose and a severe breach risk — fines reach tens of millions of EUR.
+2. **Lawful basis.** Usually **legitimate interest** for public-data scraping — but it requires a balancing test (your interest vs. the person's reasonable expectations). Document it. See `../gdpr-privacy/SKILL.md` for the LIA.
+3. **Minimization.** Collect only the fields the purpose needs. Drop everything else at extraction, not later.
+4. **Special categories.** Health, politics, religion, sexual orientation, biometrics (Art. 9) — filter these out at the source. Do not collect them speculatively.
+5. **Objection signals.** Honor robots.txt / ai.txt objections as evidence of respecting data-subject expectations.
+
+## Minimization & retention checklist
+
+- [ ] Field list maps 1:1 to the stated purpose — no "just in case" columns.
+- [ ] Special categories filtered at extraction.
+- [ ] Retention period set, with a deletion trigger (not "keep forever").
+- [ ] Source URL + scrape timestamp recorded per record for provenance.
+- [ ] A human owner signed off on the basis if any personal data is involved.
+
+## Case summaries (2024-2025)
+
+- **hiQ v. LinkedIn** — scraping *public* data is not automatically a CFAA violation. The floor, not a license; everything else (auth, contract, GDPR) still applies.
+- **Meta v. Bright Data (2024)** — the exposure is **breach of contract** when you accepted ToS, typically by logging in. Logged-out public scraping weakens the contract claim. Lesson: stay logged-out, never bypass auth.
+- **Reddit v. Perplexity AI (2025)** — centers on whether **rate limits and anti-bot measures were circumvented**, distinct from the public-vs-private axis. Bypassing a control you were shown (CAPTCHA, hard block, enforced limit) is materially worse than pacing public pages. Lesson: pace, do not circumvent.
+
+Sources: groupbwt.com "Web Scraping Legal Issues: 2025 Enterprise Compliance Guide"; sociavault.com court-case roundup; promptcloud.com 2026 compliance guide; medium.com/deep-tech-insights "The €20 Million GDPR Mistake" (2025); zyte.com AI personal-data scraping guidance. All accessed 2026-06-02.