snoopscan 0.1.0__tar.gz

snoopscan-0.1.0/.gitignore

@@ -0,0 +1,31 @@
+# Secrets and local config — constraint C3: nothing identifying in the repo.
+.env
+.env.*
+!.env.example
+*.pem
+*.key
+
+# Lead exports are real people's contact details. They are the output of a
+# run, never source, and a repo is exactly the wrong place for them.
+exports/
+*.csv
+
+# Python
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+*.egg-info/
+dist/
+build/
+
+# Tooling
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+licences.json
+
+# OS
+.DS_Store

snoopscan-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

snoopscan-0.1.0/PKG-INFO

@@ -0,0 +1,118 @@
+Metadata-Version: 2.5
+Name: snoopscan
+Version: 0.1.0
+Summary: Python client for the SnoopScan web scraping API
+Project-URL: Homepage, https://snoopscan.com
+Project-URL: Documentation, https://snoopscan.com/docs
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,crawler,llm,markdown,scraping,web-scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Description-Content-Type: text/markdown
+
+# snoopscan
+
+Python client for the SnoopScan scraping snoop.
+
+MIT licensed. The server is AGPL-3.0; a client library must not be, or every
+application that imports it inherits the copyleft.
+
+```bash
+pip install snoopscan
+```
+
+```python
+from snoopscan import SnoopScan
+
+snoop = SnoopScan(api_key="sk_...")
+
+page = snoop.scrape("https://example.com/")
+print(page.markdown)
+
+for link in snoop.map("https://example.com/", limit=100):
+    print(link.url)
+
+job = snoop.crawl_and_wait("https://example.com/", limit=50)
+print(job.completed, "of", job.total)
+```
+
+## Platforms
+
+A site that publishes its data as JSON is asked, not crawled:
+
+```python
+catalogue = snoop.products("https://store.example.com")      # Shopify, WooCommerce, Squarespace, Magento
+for p in catalogue["products"]:
+    print(p["title"], p["price"], p["currency"], p["available"])
+
+posts = snoop.posts("https://blog.example.com")               # WordPress, Substack, Squarespace, Discourse; else the feed
+print(posts["source"], len(posts["posts"]))
+```
+
+Every page's `metadata.platform` says what built it, and a `scrape()` of a
+Shopify, WooCommerce or Amazon product page carries `product` beside the
+markdown. Amazon shows the honest client no price — pass `tier="browser"`.
+
+## Monitors
+
+```python
+m = snoop.create_monitor("Pricing", ["https://example.com/pricing"], intervalMinutes=60,
+                         webhook="https://hooks.example.com/snoop")
+check = snoop.run_monitor(m["id"])          # a check now: {"counts": {...}, "pages": [...]}
+snoop.monitor_checks(m["id"])                # recent checks
+snoop.delete_monitor(m["id"])
+```
+
+Each page in a check is `same`, `changed` (with a git diff), `new` or `error`.
+The webhook `monitor.check.completed` fires only when a check has something to
+say.
+
+## Base URL
+
+Defaults to `http://localhost:8099`, the engine's own dev port. Point it
+elsewhere with the `SNOOP_BASE_URL` environment variable, or per client:
+
+```python
+snoop = SnoopScan(api_key="sk_...", base_url="https://api.example.com")
+```
+
+## Errors
+
+Every failure raises `SnoopScanError` carrying the API's machine-readable
+code, so callers can branch on what actually happened:
+
+```python
+from snoopscan import SnoopScanError
+
+try:
+    page = snoop.scrape(url)
+except SnoopScanError as exc:
+    if exc.is_blocked:
+        ...          # BLOCKED — the target refused us; retrying as-is will not help
+    elif exc.code == "FETCH_FAILED":
+        ...          # the target could not be reached; retrying may help
+    elif exc.code == "INVALID_REQUEST":
+        ...          # our request was wrong; fix it, do not retry
+```
+
+## Cost
+
+Every response carries what it cost to produce — the tier that answered, every
+tier attempted, proxy bytes, browser milliseconds, and whether it came from
+cache. A cache hit reports the accounting of the fetch that filled it, so
+`cost.tier` is never null on a page that was really fetched once.
+
+## Development
+
+From a checkout of the engine repo:
+
+```bash
+uv pip install -e sdk/python --python .venv/bin/python
+```

snoopscan-0.1.0/README.md

@@ -0,0 +1,99 @@
+# snoopscan
+
+Python client for the SnoopScan scraping snoop.
+
+MIT licensed. The server is AGPL-3.0; a client library must not be, or every
+application that imports it inherits the copyleft.
+
+```bash
+pip install snoopscan
+```
+
+```python
+from snoopscan import SnoopScan
+
+snoop = SnoopScan(api_key="sk_...")
+
+page = snoop.scrape("https://example.com/")
+print(page.markdown)
+
+for link in snoop.map("https://example.com/", limit=100):
+    print(link.url)
+
+job = snoop.crawl_and_wait("https://example.com/", limit=50)
+print(job.completed, "of", job.total)
+```
+
+## Platforms
+
+A site that publishes its data as JSON is asked, not crawled:
+
+```python
+catalogue = snoop.products("https://store.example.com")      # Shopify, WooCommerce, Squarespace, Magento
+for p in catalogue["products"]:
+    print(p["title"], p["price"], p["currency"], p["available"])
+
+posts = snoop.posts("https://blog.example.com")               # WordPress, Substack, Squarespace, Discourse; else the feed
+print(posts["source"], len(posts["posts"]))
+```
+
+Every page's `metadata.platform` says what built it, and a `scrape()` of a
+Shopify, WooCommerce or Amazon product page carries `product` beside the
+markdown. Amazon shows the honest client no price — pass `tier="browser"`.
+
+## Monitors
+
+```python
+m = snoop.create_monitor("Pricing", ["https://example.com/pricing"], intervalMinutes=60,
+                         webhook="https://hooks.example.com/snoop")
+check = snoop.run_monitor(m["id"])          # a check now: {"counts": {...}, "pages": [...]}
+snoop.monitor_checks(m["id"])                # recent checks
+snoop.delete_monitor(m["id"])
+```
+
+Each page in a check is `same`, `changed` (with a git diff), `new` or `error`.
+The webhook `monitor.check.completed` fires only when a check has something to
+say.
+
+## Base URL
+
+Defaults to `http://localhost:8099`, the engine's own dev port. Point it
+elsewhere with the `SNOOP_BASE_URL` environment variable, or per client:
+
+```python
+snoop = SnoopScan(api_key="sk_...", base_url="https://api.example.com")
+```
+
+## Errors
+
+Every failure raises `SnoopScanError` carrying the API's machine-readable
+code, so callers can branch on what actually happened:
+
+```python
+from snoopscan import SnoopScanError
+
+try:
+    page = snoop.scrape(url)
+except SnoopScanError as exc:
+    if exc.is_blocked:
+        ...          # BLOCKED — the target refused us; retrying as-is will not help
+    elif exc.code == "FETCH_FAILED":
+        ...          # the target could not be reached; retrying may help
+    elif exc.code == "INVALID_REQUEST":
+        ...          # our request was wrong; fix it, do not retry
+```
+
+## Cost
+
+Every response carries what it cost to produce — the tier that answered, every
+tier attempted, proxy bytes, browser milliseconds, and whether it came from
+cache. A cache hit reports the accounting of the fetch that filled it, so
+`cost.tier` is never null on a page that was really fetched once.
+
+## Development
+
+From a checkout of the engine repo:
+
+```bash
+uv pip install -e sdk/python --python .venv/bin/python
+```

snoopscan-0.1.0/pyproject.toml

@@ -0,0 +1,37 @@
+[project]
+name = "snoopscan"
+version = "0.1.0"
+description = "Python client for the SnoopScan web scraping API"
+readme = "README.md"
+requires-python = ">=3.10"
+# MIT, NOT AGPL. The server is AGPL; a client library must not be, or every
+# application that imports it inherits the copyleft. An AGPL SDK is an
+# adoption blocker, which is the opposite of what a client is for.
+license = "MIT"
+license-files = ["LICENSE"]
+dependencies = ["httpx>=0.27"]
+keywords = ["scraping", "crawler", "web-scraping", "markdown", "llm", "agents"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Text Processing :: Markup :: Markdown",
+]
+
+[project.scripts]
+# The command IS the package name: `snoop` is taken on both PyPI and npm, and
+# a short alias like `ss` would shadow a real tool on Linux.
+snoopscan = "snoopscan.cli:main"
+
+[project.urls]
+Homepage = "https://snoopscan.com"
+Documentation = "https://snoopscan.com/docs"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["snoopscan"]

snoopscan-0.1.0/snoopscan/__init__.py

@@ -0,0 +1,33 @@
+"""Python client for the SnoopScan web scraping API."""
+
+from snoopscan.client import (
+    AsyncSnoopScan,
+    Cost,
+    CrawlJob,
+    Document,
+    SnoopScan,
+    SnoopScanError,
+)
+
+__all__ = [
+    "AsyncSnoopScan",
+    "Cost",
+    "CrawlJob",
+    "Document",
+    "SnoopScan",
+    "SnoopScanError",
+]
+
+def _version() -> str:
+    """From the installed distribution, so pyproject.toml is the only place the
+    number is written. A client that reports a version it is not is worse than
+    one that says it does not know."""
+    from importlib.metadata import PackageNotFoundError, version
+
+    try:
+        return version("snoopscan")
+    except PackageNotFoundError:
+        return "0.0.0+unknown"
+
+
+__version__ = _version()

snoopscan-0.1.0/snoopscan/cli.py

@@ -0,0 +1,419 @@
+"""The `snoopscan` command line.
+
+One vocabulary across the REST API, the SDK and this: `scrape` here is
+`/v1/scrape` there is `client.scrape()` in Python. Learning one surface teaches
+the other two, and a docs example pastes into any of them.
+
+Two things this deliberately does that a wrapper usually does not:
+
+**It tells you what it cost and how it got there.** Every result prints the
+tier that answered and the credits spent. A scraper that hides whether a page
+came from a plain HTTP request or forty seconds of stealth browser is hiding
+the only number that predicts the bill.
+
+**It distinguishes the ways a page can fail.** BLOCKED, THIN, TARGET_ERROR and
+ROBOTS_DENIED are different problems with different fixes, and collapsing them
+into "failed" is how a formatting bug gets mistaken for a WAF.
+
+Standard library only: argparse, not click. A CLI that drags a dependency tree
+into every project that installs the SDK is a tax on people who only wanted the
+client.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from typing import Any
+
+from .client import DEFAULT_BASE_URL, SnoopScan, SnoopScanError
+
+ENV_KEY = "SNOOPSCAN_API_KEY"
+ENV_URL = "SNOOPSCAN_BASE_URL"
+
+# Exit codes, so a shell script can branch. 1 is "we failed", 2 is "the target
+# refused us" — different problems, and a caller retrying the second one
+# forever is exactly the waste this separation prevents.
+EXIT_OK = 0
+EXIT_ERROR = 1
+EXIT_BLOCKED = 2
+
+
+# --------------------------------------------------------------------------
+# Output
+# --------------------------------------------------------------------------
+
+
+def _emit(value: Any, args: argparse.Namespace) -> None:
+    """Write the result, to a file when asked, to stdout otherwise."""
+    if isinstance(value, str):
+        text = value
+    else:
+        text = json.dumps(value, indent=2 if args.pretty else None, ensure_ascii=False)
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as handle:
+            handle.write(text)
+        _note(f"written to {args.output} ({len(text):,} chars)", args)
+        return
+    print(text)
+
+
+def _note(message: str, args: argparse.Namespace) -> None:
+    """A line for the operator, on stderr so it never pollutes piped output."""
+    if not args.quiet:
+        print(message, file=sys.stderr)
+
+
+def _receipt(payload: dict[str, Any], args: argparse.Namespace) -> None:
+    """How the page was got, and what it cost.
+
+    On stderr deliberately: `snoopscan scrape url > page.md` must produce a
+    clean file, and the receipt is for the human watching, not the pipe.
+    """
+    if args.quiet or not isinstance(payload, dict):
+        return
+    cost = payload.get("cost") or {}
+    meta = payload.get("metadata") or {}
+    bits = []
+    if tier := (cost.get("tier") or meta.get("tier")):
+        bits.append(f"tier={tier}")
+    if (credits := cost.get("credits")) is not None:
+        bits.append(f"credits={credits}")
+    if (ms := cost.get("durationMs") or cost.get("duration_ms")) is not None:
+        bits.append(f"{int(ms) / 1000:.1f}s")
+    if bits:
+        print("  " + "  ".join(bits), file=sys.stderr)
+
+
+# --------------------------------------------------------------------------
+# Commands
+# --------------------------------------------------------------------------
+
+
+def _options(args: argparse.Namespace) -> dict[str, Any]:
+    """Shared scrape-ish options, omitting anything the caller left alone so
+    the API applies its own defaults rather than ours."""
+    out: dict[str, Any] = {}
+    if args.tier:
+        out["tier"] = args.tier
+    if args.timeout:
+        out["timeout"] = args.timeout
+    if getattr(args, "max_age", None) is not None:
+        out["maxAge"] = args.max_age
+    if getattr(args, "formats", None):
+        out["formats"] = args.formats.split(",")
+    return out
+
+
+def cmd_scrape(client: SnoopScan, args: argparse.Namespace) -> int:
+    doc = client.scrape(args.url, **_options(args))
+    _receipt(doc.raw, args)
+    if doc.is_suspect and not args.quiet:
+        # Worth saying out loud: a low-confidence extraction reads like a
+        # normal result and is the shape a menu bar or a nav shell arrives in.
+        print(f"  low confidence ({doc.extraction_confidence:.2f}) — check it", file=sys.stderr)
+    _emit(doc.raw if args.json else (doc.markdown or ""), args)
+    return EXIT_OK
+
+
+def cmd_map(client: SnoopScan, args: argparse.Namespace) -> int:
+    links = client.map(args.url, **_options(args))
+    _note(f"  {len(links)} links", args)
+    if args.json:
+        _emit(links, args)
+    else:
+        _emit("\n".join(str(row.get("url", row)) for row in links), args)
+    return EXIT_OK
+
+
+def cmd_search(client: SnoopScan, args: argparse.Namespace) -> int:
+    body: dict[str, Any] = {"limit": args.limit}
+    if args.scrape:
+        body["scrapeResults"] = True
+    data = client.search(args.query, **body)
+    results = data.get("results", []) if isinstance(data, dict) else data
+    # Which provider answered. The ladder falling through to a weaker engine is
+    # correct behaviour and invisible unless it is printed — a search that
+    # silently degrades returns a worse source mix at the same speed.
+    if isinstance(data, dict) and (provider := data.get("provider")):
+        _note(f"  provider={provider}  results={len(results)}", args)
+    if args.json:
+        _emit(data, args)
+    else:
+        _emit("\n".join(f"{r.get('title', '')}\n  {r.get('url', '')}" for r in results), args)
+    return EXIT_OK
+
+
+def cmd_crawl(client: SnoopScan, args: argparse.Namespace) -> int:
+    options = _options(args)
+    if args.limit:
+        options["limit"] = args.limit
+    if args.wait:
+        docs = client.crawl_and_wait(args.url, max_wait=args.max_wait, **options)
+        _note(f"  {len(docs)} pages", args)
+        joined = "\n\n---\n\n".join(d.markdown or "" for d in docs)
+        _emit([d.raw for d in docs] if args.json else joined, args)
+        return EXIT_OK
+    job = client.crawl(args.url, **options)
+    _note(f"  job {job.id} started — snoopscan crawl-status {job.id}", args)
+    _emit({"id": job.id, "status": job.status}, args)
+    return EXIT_OK
+
+
+def cmd_crawl_status(client: SnoopScan, args: argparse.Namespace) -> int:
+    job = client.crawl_status(args.job_id)
+    _emit(
+        {
+            "id": job.id,
+            "status": job.status,
+            "total": job.total,
+            "completed": job.completed,
+            "failed": job.failed,
+        },
+        args,
+    )
+    return EXIT_OK
+
+
+def cmd_extract(client: SnoopScan, args: argparse.Namespace) -> int:
+    schema = json.loads(args.schema) if args.schema.strip().startswith("{") else json.loads(
+        open(args.schema, encoding="utf-8").read()
+    )
+    rows = client.extract(args.urls, schema, prompt=args.prompt)
+    _emit(rows, args)
+    return EXIT_OK
+
+
+def cmd_parse(client: SnoopScan, args: argparse.Namespace) -> int:
+    if args.target.startswith(("http://", "https://")):
+        data = client.parse(url=args.target)
+    else:
+        with open(args.target, encoding="utf-8", errors="replace") as handle:
+            data = client.parse(content=handle.read())
+    _receipt(data, args)
+    _emit(data if args.json else data.get("markdown", ""), args)
+    return EXIT_OK
+
+
+def _listing(rows: list[dict[str, Any]], title_keys: tuple[str, ...]) -> str:
+    """A readable line per item.
+
+    The default output of a catalogue command must not be the catalogue. One
+    blog returned 4.2 MB of post bodies to a terminal — technically the right
+    data, unusably delivered. Full fidelity is one flag away (`--json`), and
+    `-o` writes it to a file where that size belongs.
+    """
+    lines = []
+    for row in rows:
+        title = next((str(row[k]) for k in title_keys if row.get(k)), "(untitled)")
+        url = row.get("url") or row.get("link") or ""
+        price = row.get("price")
+        suffix = f"  {price}" if price else ""
+        lines.append(f"{title}{suffix}\n  {url}")
+    return "\n".join(lines)
+
+
+def cmd_products(client: SnoopScan, args: argparse.Namespace) -> int:
+    data = client.products(args.url)
+    products = data.get("products", [])
+    _note(f"  platform={data.get('platform')}  total={data.get('total')}  returned={len(products)}", args)
+    _emit(data if args.json else _listing(products, ("title", "name")), args)
+    return EXIT_OK
+
+
+def cmd_posts(client: SnoopScan, args: argparse.Namespace) -> int:
+    data = client.posts(args.url)
+    posts = data.get("posts", [])
+    _note(f"  platform={data.get('platform')}  source={data.get('source')}  returned={len(posts)}", args)
+    _emit(data if args.json else _listing(posts, ("title", "name")), args)
+    return EXIT_OK
+
+
+def cmd_monitor(client: SnoopScan, args: argparse.Namespace) -> int:
+    if args.action == "list":
+        _emit(client.monitors(), args)
+    elif args.action == "get":
+        _emit(client.monitor(args.monitor_id), args)
+    elif args.action == "run":
+        _emit(client.run_monitor(args.monitor_id), args)
+    elif args.action == "checks":
+        _emit(client.monitor_checks(args.monitor_id), args)
+    elif args.action == "delete":
+        client.delete_monitor(args.monitor_id)
+        _note(f"  deleted {args.monitor_id}", args)
+    elif args.action == "create":
+        _emit(
+            client.create_monitor(
+                args.name, args.urls.split(","), intervalMinutes=args.interval, goal=args.goal
+            ),
+            args,
+        )
+    return EXIT_OK
+
+
+def cmd_status(client: SnoopScan, args: argparse.Namespace) -> int:
+    """Whether this is configured AND whether the engine can do the hard work.
+
+    The second half matters: Camoufox's browser build lives in an OS cache
+    directory that cleaning tools empty, and losing it drops the only rungs
+    that pass a hard anti-bot check. The engine then answers BLOCKED for those
+    domains, which reads as the targets refusing us rather than as a missing
+    dependency. One line here beats an hour of misdirected debugging.
+    """
+    import httpx
+
+    base = client.base_url
+    print(f"  api      {base}")
+    try:
+        health = httpx.get(f"{base}/health", timeout=10).json()
+        print(f"  engine   {health.get('status', '?')}  v{health.get('version', '?')}")
+        tiers = health.get("tiers") or []
+        deep = health.get("deepTiersAvailable")
+        print(f"  tiers    {', '.join(tiers) if tiers else 'unknown'}")
+        if deep is False:
+            print("  WARNING  the deep rungs are missing — hard sites will report BLOCKED")
+            print("           fix: python -m camoufox fetch")
+        if degraded := health.get("searchProvidersDegraded"):
+            print(f"  WARNING  search rungs degraded: {', '.join(degraded)}")
+        if health.get("saturated"):
+            print(f"  WARNING  engine saturated (loop lag {health.get('recentLagMs')}ms)")
+    except Exception as exc:  # noqa: BLE001 - status must report, never raise
+        print(f"  engine   unreachable ({type(exc).__name__})")
+        return EXIT_ERROR
+    return EXIT_OK
+
+
+# --------------------------------------------------------------------------
+# Parser
+# --------------------------------------------------------------------------
+
+
+def build_parser() -> argparse.ArgumentParser:
+    # Global flags live on a PARENT parser so they work on either side of the
+    # subcommand. argparse's default puts them before it only, and
+    # `snoopscan scrape URL --json` — which is what everyone types — would fail
+    # with "unrecognized arguments". A CLI that rejects the natural word order
+    # is a CLI people stop using.
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("--api-key", help=f"defaults to ${ENV_KEY}")
+    common.add_argument("--base-url", help=f"defaults to ${ENV_URL} or {DEFAULT_BASE_URL}")
+    common.add_argument("-o", "--output", help="write to a file instead of stdout")
+    common.add_argument("--json", action="store_true", help="full JSON, not just the content")
+    common.add_argument("--pretty", action="store_true", help="indent the JSON")
+    common.add_argument("-q", "--quiet", action="store_true", help="no receipts on stderr")
+
+    # Flags belong AFTER the subcommand, as in git and docker: `snoopscan
+    # scrape URL --json`. They are deliberately NOT on the top-level parser —
+    # with `parents` on both, argparse re-applies the subparser's default and
+    # silently discards a flag given before the verb, which is worse than
+    # rejecting it.
+    parser = argparse.ArgumentParser(
+        prog="snoopscan",
+        description="Scrape, crawl, search and extract the web. Same verbs as the API and the SDK.",
+    )
+
+    subs = parser.add_subparsers(dest="command", required=True, parser_class=argparse.ArgumentParser)
+
+    def page_options(sub: argparse.ArgumentParser) -> None:
+        sub.add_argument("--tier", choices=["http", "impersonate", "browser", "stealth", "stealth_hard", "mobile", "auto"])
+        sub.add_argument("--timeout", type=int, help="milliseconds")
+        sub.add_argument("--max-age", type=int, help="accept a cached page this many ms old")
+        sub.add_argument("--formats", help="comma separated: markdown,html,rawHtml,links,screenshot")
+
+    s = subs.add_parser("scrape", parents=[common], help="get clean content from one URL")
+    s.add_argument("url")
+    page_options(s)
+    s.set_defaults(func=cmd_scrape)
+
+    s = subs.add_parser("crawl", parents=[common], help="crawl a site")
+    s.add_argument("url")
+    s.add_argument("--limit", type=int, help="maximum pages")
+    s.add_argument("--wait", action="store_true", help="block until it finishes")
+    s.add_argument("--max-wait", type=float, default=900.0)
+    page_options(s)
+    s.set_defaults(func=cmd_crawl)
+
+    s = subs.add_parser("crawl-status", parents=[common], help="how a crawl is going")
+    s.add_argument("job_id")
+    s.set_defaults(func=cmd_crawl_status)
+
+    s = subs.add_parser("map", parents=[common], help="discover URLs without fetching bodies")
+    s.add_argument("url")
+    page_options(s)
+    s.set_defaults(func=cmd_map)
+
+    s = subs.add_parser("search", parents=[common], help="search the web")
+    s.add_argument("query")
+    s.add_argument("--limit", type=int, default=10)
+    s.add_argument("--scrape", action="store_true", help="fetch each result too")
+    s.set_defaults(func=cmd_search)
+
+    s = subs.add_parser("extract", parents=[common], help="structured data against a schema")
+    s.add_argument("urls", nargs="+")
+    s.add_argument("--schema", required=True, help="inline JSON or a path to a .json file")
+    s.add_argument("--prompt", help="what to pull out, in words")
+    s.set_defaults(func=cmd_extract)
+
+    s = subs.add_parser("parse", parents=[common], help="a document to markdown — PDF, DOCX, XLSX, HTML")
+    s.add_argument("target", help="a URL or a local file")
+    s.set_defaults(func=cmd_parse)
+
+    s = subs.add_parser("products", parents=[common], help="a store's catalogue, from its own endpoint")
+    s.add_argument("url")
+    s.set_defaults(func=cmd_products)
+
+    s = subs.add_parser("posts", parents=[common], help="a site's posts, from its API or feed")
+    s.add_argument("url")
+    s.set_defaults(func=cmd_posts)
+
+    s = subs.add_parser("monitor", parents=[common], help="watch pages for changes")
+    s.add_argument("action", choices=["list", "get", "run", "checks", "delete", "create"])
+    s.add_argument("monitor_id", nargs="?")
+    s.add_argument("--name")
+    s.add_argument("--urls", help="comma separated")
+    s.add_argument("--interval", type=int, default=60, help="minutes, minimum 5")
+    s.add_argument("--goal", help="what change matters")
+    s.set_defaults(func=cmd_monitor)
+
+    s = subs.add_parser("status", parents=[common], help="is this configured, and can the engine do the hard work")
+    s.set_defaults(func=cmd_status)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    api_key = args.api_key or os.environ.get(ENV_KEY, "")
+    base_url = args.base_url or os.environ.get(ENV_URL) or DEFAULT_BASE_URL
+    if not api_key and args.command != "status":
+        print(
+            f"No API key. Pass --api-key or set {ENV_KEY}.",
+            file=sys.stderr,
+        )
+        return EXIT_ERROR
+
+    client = SnoopScan(api_key or "none", base_url=base_url)
+    try:
+        return int(args.func(client, args))
+    except SnoopScanError as exc:
+        # The distinction is the point. A target refusing us is not the same
+        # problem as a thin page or a broken request, and a caller that retries
+        # all three identically wastes money on the two that will never change.
+        # `str(exc)` already carries the code, so printing it again reads
+        # "BLOCKED: BLOCKED: ...".
+        print(str(exc), file=sys.stderr)
+        if exc.detail and not args.quiet:
+            print(f"  detail: {json.dumps(exc.detail)[:300]}", file=sys.stderr)
+        return EXIT_BLOCKED if exc.code in {"BLOCKED", "ROBOTS_DENIED"} else EXIT_ERROR
+    except KeyboardInterrupt:
+        return EXIT_ERROR
+    finally:
+        client.close()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

snoopscan-0.1.0/snoopscan/client.py

@@ -0,0 +1,473 @@
+"""Python client for the SnoopScan web scraping API.
+
+Deliberately mirrors the shape of the established clients in this space —
+`scrape`, `crawl`, `map`, `extract`, `search`, plus a blocking `crawl_and_wait`
+— because the migration promise is that existing code changes a base URL and
+keeps working. Method and option names are the API's names.
+
+Written from our own OpenAPI surface, not from any other client's source
+(constraint C1).
+
+    from snoopscan import SnoopScan
+
+    snoop = SnoopScan(api_key="sk_...")
+    page = snoop.scrape("https://example.com")
+    print(page.markdown)
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import httpx
+
+# The engine's documented dev port (README, .claude/launch.json). This used to
+# be 8000, which no deployment of ours listens on, so every local caller had to
+# pass `base_url` and the first call of a session failed with a connection
+# error (measured, Sep 2026). `SNOOP_BASE_URL` overrides it, so pointing
+# a script at staging or production is an environment variable, not a code edit.
+# `SNOOP_BASE_URL` is the pre-rename spelling, still honoured so an existing
+# deployment does not break. The CLI reads `SNOOPSCAN_BASE_URL`, and before
+# this the two disagreed: setting the documented variable did nothing to the
+# SDK, which is the worst kind of configuration bug — silent and correct-looking.
+#
+# The default is localhost because the engine is self-hosted; a published
+# client cannot guess where YOUR instance runs. Set SNOOPSCAN_BASE_URL.
+DEFAULT_BASE_URL = (
+    os.environ.get("SNOOPSCAN_BASE_URL")
+    or os.environ.get("SNOOP_BASE_URL")
+    or "http://localhost:8099"
+)
+DEFAULT_TIMEOUT = 120.0
+
+
+class SnoopScanError(Exception):
+    """An error returned by the API.
+
+    Carries the machine-readable code so callers can branch on it —
+    `BLOCKED` and `TARGET_ERROR` mean different things and deserve different
+    handling.
+    """
+
+    def __init__(self, code: str, message: str, detail: dict[str, Any] | None = None) -> None:
+        super().__init__(f"{code}: {message}")
+        self.code = code
+        self.message = message
+        self.detail = detail or {}
+
+    @property
+    def is_blocked(self) -> bool:
+        return self.code == "BLOCKED"
+
+    @property
+    def is_target_error(self) -> bool:
+        """The site responded, but not with the page — a genuine 404 or 5xx,
+        not us being blocked. Retrying will not help."""
+        return self.code == "TARGET_ERROR"
+
+    @property
+    def is_rate_limited(self) -> bool:
+        return self.code == "RATE_LIMITED"
+
+
+@dataclass
+class Cost:
+    """What the request actually cost. Failed requests carry no cost."""
+
+    tier: str | None = None
+    tiers_attempted: list[str] = field(default_factory=list)
+    proxy_used: bool = False
+    proxy_type: str | None = None
+    proxy_bytes: int = 0
+    browser_ms: int = 0
+    extraction_path: str | None = None
+    cached: bool = False
+
+
+@dataclass
+class Document:
+    markdown: str | None = None
+    html: str | None = None
+    raw_html: str | None = None
+    links: list[str] = field(default_factory=list)
+    json_: dict[str, Any] | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    cost: Cost = field(default_factory=Cost)
+    # The payload as it arrived. Kept because the typed fields are a curated
+    # view: anything the API adds later, and anything a caller wants to dump
+    # verbatim, is otherwise lost the moment it is parsed.
+    raw: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def title(self) -> str | None:
+        return self.metadata.get("title")
+
+    @property
+    def url(self) -> str | None:
+        return self.metadata.get("url")
+
+    @property
+    def page_type(self) -> str:
+        return self.metadata.get("pageType", "unknown")
+
+    @property
+    def word_count(self) -> int:
+        return int(self.metadata.get("wordCount", 0))
+
+    @property
+    def extraction_confidence(self) -> float:
+        """0-1. Treat anything below 0.5 as suspect and cross-check it."""
+        return float(self.metadata.get("extractionConfidence", 0.0))
+
+    @property
+    def is_suspect(self) -> bool:
+        return self.extraction_confidence < 0.5
+
+    @classmethod
+    def from_payload(cls, data: dict[str, Any]) -> Document:
+        return cls(
+            markdown=data.get("markdown"),
+            html=data.get("html"),
+            raw_html=data.get("rawHtml"),
+            links=data.get("links") or [],
+            json_=data.get("json"),
+            metadata=data.get("metadata") or {},
+            cost=Cost(
+                **{k: v for k, v in (data.get("cost") or {}).items() if k in Cost.__annotations__}
+            ),
+            raw=data,
+        )
+
+
+@dataclass
+class CrawlJob:
+    id: str
+    status: str
+    total: int = 0
+    completed: int = 0
+    failed: int = 0
+    cost: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def finished(self) -> bool:
+        return self.status in ("completed", "failed", "cancelled")
+
+
+class SnoopScan:
+    """Synchronous client. See AsyncSnoopScan for the async form."""
+
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: float = DEFAULT_TIMEOUT,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self._client = httpx.Client(
+            timeout=timeout,
+            headers={
+                "Authorization": f"Bearer {api_key}",
+                "Content-Type": "application/json",
+            },
+        )
+
+    # -- plumbing ---------------------------------------------------------
+
+    def _post(self, path: str, body: dict[str, Any]) -> Any:
+        return self._unwrap(self._client.post(f"{self.base_url}{path}", json=body))
+
+    def _get(self, path: str, params: dict[str, Any] | None = None) -> Any:
+        return self._unwrap(self._client.get(f"{self.base_url}{path}", params=params))
+
+    def _delete(self, path: str) -> Any:
+        return self._unwrap(self._client.delete(f"{self.base_url}{path}"))
+
+    @staticmethod
+    def _unwrap(response: httpx.Response) -> Any:
+        try:
+            payload = response.json()
+        except ValueError:
+            # Never leak an httpx exception for an API-level failure: callers
+            # branch on `SnoopScanError.code`, and a raw HTTPStatusError
+            # would bypass that entirely. A non-JSON body means something in
+            # front of the API answered — a proxy, a load balancer, an nginx
+            # error page — so it is reported as INTERNAL with the status.
+            raise SnoopScanError(
+                "INTERNAL",
+                f"Non-JSON response from the API (HTTP {response.status_code})",
+                {"status_code": response.status_code},
+            ) from None
+
+        if not payload.get("success", False):
+            error = payload.get("error") or {}
+            raise SnoopScanError(
+                error.get("code", "INTERNAL"),
+                error.get("message", "Unknown error"),
+                error.get("detail"),
+            )
+        return payload.get("data")
+
+    # -- endpoints --------------------------------------------------------
+
+    def scrape(self, url: str, **options: Any) -> Document:
+        """Fetch and extract one URL."""
+        return Document.from_payload(self._post("/v1/scrape", {"url": url, **options}))
+
+    def crawl(self, url: str, **options: Any) -> CrawlJob:
+        """Start a crawl. Returns immediately with a job id."""
+        data = self._post("/v1/crawl", {"url": url, **options})
+        return CrawlJob(id=data["id"], status=data["status"])
+
+    def crawl_status(self, job_id: str) -> CrawlJob:
+        data = self._get(f"/v1/crawl/{job_id}")
+        return CrawlJob(
+            id=data["id"],
+            status=data["status"],
+            total=data.get("total", 0),
+            completed=data.get("completed", 0),
+            failed=data.get("failed", 0),
+            cost=data.get("cost") or {},
+        )
+
+    def crawl_pages(
+        self, job_id: str, cursor: str | None = None, limit: int = 50
+    ) -> tuple[list[Document], str | None]:
+        """One page of results, plus the cursor for the next.
+
+        Results are paginated rather than inlined because a 10,000-page crawl
+        must not arrive as a single JSON body.
+        """
+        params: dict[str, Any] = {"limit": limit}
+        if cursor:
+            params["cursor"] = cursor
+        data = self._get(f"/v1/crawl/{job_id}/pages", params)
+        documents = [Document.from_payload(row) for row in data.get("pages", [])]
+        next_link = data.get("next")
+        next_cursor = next_link.split("cursor=")[-1] if next_link else None
+        return documents, next_cursor
+
+    def crawl_errors(self, job_id: str) -> list[dict[str, Any]]:
+        return self._get(f"/v1/crawl/{job_id}/errors").get("errors", [])
+
+    def cancel_crawl(self, job_id: str) -> CrawlJob:
+        data = self._client.delete(f"{self.base_url}/v1/crawl/{job_id}")
+        payload = self._unwrap(data)
+        return CrawlJob(id=payload["id"], status=payload["status"])
+
+    def crawl_and_wait(
+        self,
+        url: str,
+        poll_interval: float = 3.0,
+        max_wait: float = 900.0,
+        **options: Any,
+    ) -> list[Document]:
+        """Start a crawl and block until it finishes, then return every page.
+
+        Convenience only. For anything large, start the crawl and read pages
+        as they arrive rather than holding the whole result in memory.
+        """
+        job = self.crawl(url, **options)
+        deadline = time.monotonic() + max_wait
+
+        while time.monotonic() < deadline:
+            current = self.crawl_status(job.id)
+            if current.finished:
+                break
+            time.sleep(poll_interval)
+        else:
+            raise SnoopScanError(
+                "TIMEOUT", f"Crawl {job.id} did not finish within {max_wait:.0f}s"
+            )
+
+        documents: list[Document] = []
+        cursor: str | None = None
+        while True:
+            page, cursor = self.crawl_pages(job.id, cursor)
+            documents.extend(page)
+            if not cursor:
+                return documents
+
+    def map(self, url: str, **options: Any) -> list[dict[str, Any]]:
+        """Discover URLs without fetching page bodies. Fast and cheap."""
+        return self._post("/v1/map", {"url": url, **options}).get("links", [])
+
+    def products(self, url: str, **options: Any) -> dict[str, Any]:
+        """Every product a Shopify or WooCommerce store publishes, from its own
+        catalogue endpoint. Returns {platform, total, products, pages_fetched, cost}."""
+        return self._post("/v1/products", {"url": url, **options})
+
+    def posts(self, url: str, **options: Any) -> dict[str, Any]:
+        """A site's posts from its API (WordPress, Substack, Squarespace,
+        Discourse) or its RSS/Atom feed. Returns {platform, source, posts, cost}."""
+        return self._post("/v1/posts", {"url": url, **options})
+
+    # --- monitors: watch pages for changes on a schedule ------------------
+
+    def create_monitor(self, name: str, urls: list[str] | str, **options: Any) -> dict[str, Any]:
+        """intervalMinutes (>= 5, default 60), goal, webhook. Returns the monitor."""
+        body: dict[str, Any] = {"name": name, **options}
+        body["urls" if isinstance(urls, list) else "url"] = urls
+        return self._post("/v1/monitor", body)
+
+    def monitors(self) -> list[dict[str, Any]]:
+        return self._get("/v1/monitor").get("monitors", [])
+
+    def monitor(self, monitor_id: str) -> dict[str, Any]:
+        return self._get(f"/v1/monitor/{monitor_id}")
+
+    def delete_monitor(self, monitor_id: str) -> None:
+        self._delete(f"/v1/monitor/{monitor_id}")
+
+    def run_monitor(self, monitor_id: str) -> dict[str, Any]:
+        """A check right now; returns it."""
+        return self._post(f"/v1/monitor/{monitor_id}/run", {})
+
+    def monitor_checks(self, monitor_id: str, limit: int = 20) -> list[dict[str, Any]]:
+        out = self._get(f"/v1/monitor/{monitor_id}/checks", params={"limit": limit})
+        return out.get("checks", [])
+
+    def batch_scrape(self, urls: list[str], **options: Any) -> CrawlJob:
+        data = self._post("/v1/batch/scrape", {"urls": urls, **options})
+        return CrawlJob(id=data["id"], status=data["status"])
+
+    def extract(
+        self, urls: list[str], schema: dict[str, Any], prompt: str | None = None, **options: Any
+    ) -> list[dict[str, Any]]:
+        """Schema-constrained extraction.
+
+        Output is validated against the schema before it is returned, so a row
+        with an `error` means the page genuinely lacked the fields — not that
+        the request should be retried.
+        """
+        body: dict[str, Any] = {"urls": urls, "schema": schema, **options}
+        if prompt:
+            body["prompt"] = prompt
+        return self._post("/v1/extract", body)
+
+    def search(self, query: str, **options: Any) -> dict[str, Any]:
+        return self._post("/v1/search", {"query": query, **options})
+
+    def parse(self, *, url: str | None = None, content: str | None = None, **options: Any) -> dict[str, Any]:
+        """Turn a document into markdown — PDF, DOCX, XLSX, HTML.
+
+        Either a URL to fetch or content already in hand. The endpoint existed
+        before this method did, which meant the CLI would have had to reach
+        past the SDK to use it; one surface is the point of having an SDK.
+        """
+        body: dict[str, Any] = dict(options)
+        if url:
+            body["url"] = url
+        if content is not None:
+            body["content"] = content
+        return self._post("/v1/parse", body)
+
+    # -- lifecycle --------------------------------------------------------
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> SnoopScan:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+
+class AsyncSnoopScan:
+    """Async client. Same surface as the synchronous one."""
+
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: float = DEFAULT_TIMEOUT,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self._client = httpx.AsyncClient(
+            timeout=timeout,
+            headers={
+                "Authorization": f"Bearer {api_key}",
+                "Content-Type": "application/json",
+            },
+        )
+
+    async def _post(self, path: str, body: dict[str, Any]) -> Any:
+        response = await self._client.post(f"{self.base_url}{path}", json=body)
+        return SnoopScan._unwrap(response)
+
+    async def _get(self, path: str, params: dict[str, Any] | None = None) -> Any:
+        response = await self._client.get(f"{self.base_url}{path}", params=params)
+        return SnoopScan._unwrap(response)
+
+    async def _delete(self, path: str) -> Any:
+        return self._unwrap(await self._client.delete(f"{self.base_url}{path}"))
+
+    async def scrape(self, url: str, **options: Any) -> Document:
+        return Document.from_payload(await self._post("/v1/scrape", {"url": url, **options}))
+
+    async def crawl(self, url: str, **options: Any) -> CrawlJob:
+        data = await self._post("/v1/crawl", {"url": url, **options})
+        return CrawlJob(id=data["id"], status=data["status"])
+
+    async def crawl_status(self, job_id: str) -> CrawlJob:
+        data = await self._get(f"/v1/crawl/{job_id}")
+        return CrawlJob(
+            id=data["id"],
+            status=data["status"],
+            total=data.get("total", 0),
+            completed=data.get("completed", 0),
+            failed=data.get("failed", 0),
+            cost=data.get("cost") or {},
+        )
+
+    async def map(self, url: str, **options: Any) -> list[dict[str, Any]]:
+        return (await self._post("/v1/map", {"url": url, **options})).get("links", [])
+
+    async def products(self, url: str, **options: Any) -> dict[str, Any]:
+        return await self._post("/v1/products", {"url": url, **options})
+
+    async def posts(self, url: str, **options: Any) -> dict[str, Any]:
+        return await self._post("/v1/posts", {"url": url, **options})
+
+    async def create_monitor(
+        self, name: str, urls: list[str] | str, **options: Any
+    ) -> dict[str, Any]:
+        body: dict[str, Any] = {"name": name, **options}
+        body["urls" if isinstance(urls, list) else "url"] = urls
+        return await self._post("/v1/monitor", body)
+
+    async def monitors(self) -> list[dict[str, Any]]:
+        return (await self._get("/v1/monitor")).get("monitors", [])
+
+    async def monitor(self, monitor_id: str) -> dict[str, Any]:
+        return await self._get(f"/v1/monitor/{monitor_id}")
+
+    async def delete_monitor(self, monitor_id: str) -> None:
+        await self._delete(f"/v1/monitor/{monitor_id}")
+
+    async def run_monitor(self, monitor_id: str) -> dict[str, Any]:
+        return await self._post(f"/v1/monitor/{monitor_id}/run", {})
+
+    async def monitor_checks(self, monitor_id: str, limit: int = 20) -> list[dict[str, Any]]:
+        out = await self._get(f"/v1/monitor/{monitor_id}/checks", params={"limit": limit})
+        return out.get("checks", [])
+
+    async def extract(
+        self, urls: list[str], schema: dict[str, Any], prompt: str | None = None, **options: Any
+    ) -> list[dict[str, Any]]:
+        body: dict[str, Any] = {"urls": urls, "schema": schema, **options}
+        if prompt:
+            body["prompt"] = prompt
+        return await self._post("/v1/extract", body)
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
+
+    async def __aenter__(self) -> AsyncSnoopScan:
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.aclose()