sad-py 0.1.0__py3-none-any.whl

sad/__init__.py

@@ -0,0 +1,24 @@
+"""SAD: Statistic API Discovery. Public API version 0.1."""
+
+from .analyzer import Analyzer
+from .models import (
+    AnalysisIssue,
+    AnalysisResult,
+    Dataset,
+    Endpoint,
+    Evidence,
+    HTTPExchange,
+    Resource,
+)
+
+__version__ = "0.1.0"
+__all__ = [
+    "Analyzer",
+    "AnalysisIssue",
+    "AnalysisResult",
+    "Dataset",
+    "Endpoint",
+    "Evidence",
+    "HTTPExchange",
+    "Resource",
+]

sad/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

sad/analyzer.py

@@ -0,0 +1,255 @@
+"""Library-first analysis orchestration; all CLI and integrations use this API."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from sad.crawler import CrawlConfig, Crawler
+from sad.html import HTMLCollector
+from sad.ingest import BurpImporter, read_bounded
+from sad.javascript.engine import Engine
+from sad.javascript.parser import ASTCache
+from sad.javascript.resources import inline_source_map, references, source_map
+from sad.javascript.sourcemaps import SourceMapIndex
+from sad.models import AnalysisResult, Dataset, Endpoint, Evidence, HTTPExchange, Resource
+from sad.normalization import correlate, normalize
+from sad.protocols import SinkRegistry
+
+
+class Analyzer:
+    """Reusable analyzer. AST cache is shared; each analysis has isolated value state.
+
+    Input files and datasets are offline. Only analyze_url/analyze_urls make requests.
+    Environment substitutions are explicit; the host environment is never imported.
+    """
+
+    def __init__(
+        self,
+        *,
+        base_url: str | None = None,
+        environment: dict[str, str] | None = None,
+        registry: SinkRegistry | None = None,
+        max_steps: int = 200_000,
+        max_depth: int = 20,
+        max_resource_bytes: int = 8_000_000,
+        path_mappings: dict[str, str] | None = None,
+    ) -> None:
+        if max_steps < 1:
+            raise ValueError("max_steps must be positive")
+        self.path_mappings = path_mappings or {}
+        self.base_url = base_url
+        self.environment = environment
+        self.registry = registry or SinkRegistry()
+        self.max_steps = max_steps
+        self.max_depth = max_depth
+        self.cache = ASTCache(max_bytes=max_resource_bytes)
+
+    def analyze_js(self, source: str, *, filename: str = "input.js") -> AnalysisResult:
+        return self.analyze_dataset(Dataset(resources=[Resource(filename, source)]))
+
+    def analyze_url(self, url: str, *, config: CrawlConfig | None = None) -> AnalysisResult:
+        return self.analyze_urls([url], config=config)
+
+    def analyze_urls(self, urls: list[str], *, config: CrawlConfig | None = None) -> AnalysisResult:
+        return self.analyze_dataset(Crawler(config, cache=self.cache).collect(urls))
+
+    def analyze_exchanges(self, exchanges: list[HTTPExchange]) -> AnalysisResult:
+        return self.analyze_dataset(Dataset(exchanges=exchanges))
+
+    def analyze_path(self, path: str | Path) -> AnalysisResult:
+        root = Path(path)
+        if not root.exists():
+            raise FileNotFoundError(path)
+        dataset = Dataset()
+        files = sorted(root.rglob("*")) if root.is_dir() else [root]
+        for file in files:
+            if not file.is_file() or file.is_symlink():
+                continue
+            if any(part in ("node_modules", ".git", ".venv", "__pycache__") for part in file.parts):
+                continue
+            suffix = file.suffix.lower()
+            if suffix in (".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx", ".html", ".htm", ".map"):
+                try:
+                    content = read_bounded(file, self.cache.max_bytes).decode(
+                        "utf-8", errors="replace"
+                    )
+                    media = (
+                        "text/html"
+                        if suffix in (".html", ".htm")
+                        else "application/json"
+                        if suffix == ".map"
+                        else "application/javascript"
+                    )
+                    dataset.resources.append(Resource(file.as_posix(), content, media))
+                except (OSError, ValueError) as exc:
+                    dataset.diagnostics.append(str(exc))
+            elif suffix in (".xml", ".har", ".http", ".json"):
+                try:
+                    imported = BurpImporter.load(file, base_url=self.base_url)
+                    dataset.resources.extend(imported.resources)
+                    dataset.exchanges.extend(imported.exchanges)
+                    dataset.diagnostics.extend(imported.diagnostics)
+                except (ValueError, OSError) as exc:
+                    dataset.diagnostics.append(f"{file}: {exc}")
+        return self.analyze_dataset(dataset)
+
+    def analyze_dataset(self, dataset: Dataset) -> AnalysisResult:
+        result = AnalysisResult(diagnostics=list(dataset.diagnostics))
+        resources = list(dataset.resources)
+        for exchange in dataset.exchanges:
+            result.endpoints.append(
+                Endpoint(
+                    method=exchange.method,
+                    url=exchange.url,
+                    protocol="websocket"
+                    if exchange.request_headers.get("upgrade", "").lower() == "websocket"
+                    else "http",
+                    headers=exchange.request_headers or None,
+                    body=exchange.request_body,
+                    source_file=exchange.source_file,
+                    content_type=exchange.request_headers.get("content-type"),
+                    origin="observed",
+                    evidence=[
+                        Evidence(
+                            "captured-request",
+                            f"{exchange.method} {exchange.url}",
+                            exchange.source_file,
+                        )
+                    ],
+                    observations=[
+                        {
+                            "url": exchange.url,
+                            "method": exchange.method,
+                            "status": exchange.status,
+                            "headers": exchange.request_headers,
+                            "body": exchange.request_body,
+                        }
+                    ],
+                )
+            )
+            if exchange.response_body is not None:
+                resources.append(
+                    Resource(
+                        exchange.url,
+                        exchange.response_body,
+                        exchange.response_headers.get("content-type", ""),
+                        exchange.source_file,
+                    )
+                )
+        javascript = []
+        maps: dict[str, SourceMapIndex] = {}
+        map_aliases: dict[str, list[str]] = {}
+        origins: dict[str, str | None] = {}
+        seen: set[tuple[str, str]] = set()
+        for resource in resources:
+            key = (resource.url, resource.content)
+            if key in seen:
+                continue
+            seen.add(key)
+            if len(resource.content.encode("utf-8")) > self.cache.max_bytes:
+                result.diagnostics.append(f"resource too large: {resource.url}")
+                continue
+            source = resource.source_file or resource.url
+            base = (
+                self.base_url
+                or resource.document_url
+                or (resource.url if urlsplit(resource.url).scheme in ("http", "https") else None)
+            )
+            origins[source] = base
+            path = urlsplit(resource.url).path
+            try:
+                if "html" in resource.media_type or path.endswith((".html", ".htm")):
+                    parser = HTMLCollector(resource)
+                    parser.feed(resource.content)
+                    javascript.extend(parser.inline)
+                    result.endpoints.extend(parser.forms)
+                elif path.endswith(".map"):
+                    maps[resource.url] = SourceMapIndex(resource)
+                    maps[resource.url.removesuffix(".map")] = maps[resource.url]
+                    expanded = source_map(resource, self.cache.max_bytes)
+                    javascript.extend(expanded)
+                    for item in expanded:
+                        origins[item.source_file or item.url] = base
+                elif (
+                    "javascript" in resource.media_type
+                    or "ecmascript" in resource.media_type
+                    or path.endswith((".js", ".mjs", ".cjs", ".ts", ".jsx", ".tsx"))
+                ):
+                    javascript.append(resource)
+                    for reference in references(resource, self.cache):
+                        if reference.startswith("data:"):
+                            inline = inline_source_map(reference, self.cache.max_bytes)
+                            inline = Resource(
+                                resource.url + ".map", inline.content, inline.media_type
+                            )
+                            maps[resource.source_file or resource.url] = SourceMapIndex(inline)
+                            expanded = source_map(inline, self.cache.max_bytes)
+                            javascript.extend(expanded)
+                            for item in expanded:
+                                origins[item.source_file or item.url] = base
+                        else:
+                            map_aliases.setdefault(resource.source_file or resource.url, []).append(
+                                reference
+                            )
+            except (ValueError, RecursionError) as exc:
+                result.diagnostics.append(f"{resource.url}: {exc}")
+        for source, candidates in map_aliases.items():
+            for reference in candidates:
+                if reference in maps:
+                    maps[source] = maps[reference]
+        engine = Engine(
+            self.cache,
+            self.registry,
+            max_steps=self.max_steps,
+            max_depth=self.max_depth,
+            environment=self.environment,
+            path_mappings=self.path_mappings,
+        )
+        static = engine.analyze(javascript)
+        result.endpoints.extend(static.endpoints)
+        result.diagnostics.extend(static.diagnostics)
+        result.call_graph = static.call_graph
+        result.issues.extend(static.issues)
+        result.analysis_incomplete = static.analysis_incomplete
+        valid = []
+        for endpoint in result.endpoints:
+            try:
+                valid.append(normalize(endpoint, origins.get(endpoint.source_file, self.base_url)))
+                mapping = maps.get(endpoint.source_file)
+                location = (
+                    mapping.lookup(endpoint.source_line or 1, endpoint.source_column or 1)
+                    if mapping
+                    else None
+                )
+                if location:
+                    endpoint.evidence.append(
+                        Evidence(
+                            "source-map",
+                            f"Mapped from {endpoint.source_file}:{endpoint.source_line}",
+                            location.source_file,
+                            location.line,
+                            location.column,
+                        )
+                    )
+                    endpoint.source_file, endpoint.source_line, endpoint.source_column = (
+                        location.source_file,
+                        location.line,
+                        location.column,
+                    )
+            except ValueError as exc:
+                result.diagnostics.append(f"Invalid endpoint {endpoint.url!r}: {exc}")
+        result.endpoints = correlate(valid)
+        for diagnostic in list(result.diagnostics):
+            if not any(issue.reason == diagnostic for issue in result.issues):
+                result.add_issue("input-incomplete", diagnostic)
+        result.diagnostics = sorted(set(result.diagnostics))
+        result.issues.sort(
+            key=lambda issue: (
+                issue.source_file or "",
+                issue.source_line or 0,
+                issue.code,
+                issue.reason,
+            )
+        )
+        return result

sad/cli.py

@@ -0,0 +1,74 @@
+"""Thin command-line frontend; exit 2 for input errors, 1 with --strict diagnostics."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from sad import Analyzer, __version__
+from sad.crawler import CrawlConfig
+from sad.ingest import BurpImporter, read_bounded
+from sad.output import render
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="sad", description="Static frontend API discovery")
+    parser.add_argument("--version", action="version", version=__version__)
+    commands = parser.add_subparsers(dest="command", required=True)
+    for command in ("analyze", "analyze-js", "import-burp", "crawl"):
+        sub = commands.add_parser(command)
+        sub.add_argument("input", nargs="+" if command == "crawl" else None)
+        sub.add_argument("--base-url")
+        sub.add_argument("--format", choices=("json", "jsonl", "csv", "table"), default="table")
+        sub.add_argument("-o", "--output", type=Path)
+        sub.add_argument(
+            "--max-steps",
+            type=int,
+            default=200_000,
+            help="global static-analysis step budget, shared fairly among supplied sources",
+        )
+        sub.add_argument(
+            "--strict", action="store_true", help="exit 1 if any diagnostic is emitted"
+        )
+        if command == "crawl":
+            sub.add_argument("--depth", type=int, default=2)
+            sub.add_argument("--max-pages", type=int, default=30)
+            sub.add_argument("--max-resources", type=int, default=300)
+            sub.add_argument("--allow-origin", action="append", default=[])
+            sub.add_argument("--exclude", action="append", default=[])
+    args = parser.parse_args(argv)
+    try:
+        analyzer = Analyzer(base_url=args.base_url, max_steps=args.max_steps)
+        if args.command == "crawl":
+            result = analyzer.analyze_urls(
+                args.input,
+                config=CrawlConfig(
+                    depth=args.depth,
+                    max_pages=args.max_pages,
+                    max_resources=args.max_resources,
+                    allowed_origins=tuple(args.allow_origin),
+                    exclusions=tuple(args.exclude),
+                ),
+            )
+        elif args.command == "import-burp":
+            result = analyzer.analyze_dataset(BurpImporter.load(args.input, base_url=args.base_url))
+        elif args.command == "analyze-js":
+            result = analyzer.analyze_js(
+                read_bounded(args.input).decode("utf-8", errors="replace"), filename=args.input
+            )
+        else:
+            result = analyzer.analyze_path(args.input)
+        output = render(result, args.format)
+        if args.output:
+            args.output.write_text(output, encoding="utf-8")
+        else:
+            sys.stdout.write(output)
+        if result.analysis_incomplete:
+            print("sad: analysis incomplete; results are partial", file=sys.stderr)
+        for diagnostic in result.diagnostics:
+            print(f"sad: {diagnostic}", file=sys.stderr)
+        return 1 if args.strict and result.diagnostics else 0
+    except (OSError, ValueError) as exc:
+        print(f"sad: {exc}", file=sys.stderr)
+        return 2

sad/crawler/__init__.py

@@ -0,0 +1,192 @@
+"""Bounded resource collection with scope-checked canonical seed redirects."""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+from fnmatch import fnmatch
+from urllib.parse import urljoin, urlsplit
+
+import httpx
+
+from sad.html import HTMLCollector
+from sad.javascript.parser import ASTCache
+from sad.javascript.resources import references
+from sad.models import Dataset, Resource
+from sad.normalization import normalize_url
+
+
+def origin(url: str) -> tuple[str, str | None, int | None]:
+    parts = urlsplit(url)
+    return parts.scheme, parts.hostname, parts.port or (443 if parts.scheme == "https" else 80)
+
+
+@dataclass(frozen=True)
+class CrawlConfig:
+    depth: int = 2
+    max_pages: int = 30
+    max_resources: int = 300
+    max_bytes: int = 8_000_000
+    max_total_bytes: int = 64_000_000
+    timeout: float = 10.0
+    allowed_origins: tuple[str, ...] = ()
+    exclusions: tuple[str, ...] = ()
+    max_redirects: int = 5
+
+    def __post_init__(self) -> None:
+        if (
+            self.depth < 0
+            or min(self.max_pages, self.max_resources, self.max_bytes, self.max_total_bytes) < 1
+        ):
+            raise ValueError("crawl limits must be positive; depth may be zero")
+        if self.timeout <= 0 or self.max_redirects < 0:
+            raise ValueError("invalid timeout or redirect limit")
+
+
+class Crawler:
+    def __init__(
+        self,
+        config: CrawlConfig | None = None,
+        *,
+        client: httpx.Client | None = None,
+        cache: ASTCache | None = None,
+    ) -> None:
+        self.config = config or CrawlConfig()
+        self.client = client
+        self.cache = cache or ASTCache()
+
+    def collect(self, seeds: list[str]) -> Dataset:
+        cfg = self.config
+        for seed in seeds:
+            if urlsplit(seed).scheme not in ("http", "https"):
+                raise ValueError("crawl seeds must be HTTP(S) URLs")
+        allowed = {origin(s) for s in [*seeds, *cfg.allowed_origins]}
+        seed_urls = {normalize_url(s) for s in seeds}
+
+        def canonical_redirect(source: str, target: str) -> bool:
+            source_scheme, source_host, source_port = origin(source)
+            target_scheme, target_host, target_port = origin(target)
+            return (
+                bool(source_host and target_host)
+                and (
+                    source_host == target_host
+                    or target_host == f"www.{source_host}"
+                    or source_host == f"www.{target_host}"
+                )
+                and (
+                    source_scheme == target_scheme
+                    or (source_scheme, target_scheme) == ("http", "https")
+                )
+                and source_port == (443 if source_scheme == "https" else 80)
+                and target_port == (443 if target_scheme == "https" else 80)
+                and target_scheme in ("http", "https")
+                and not urlsplit(target).username
+                and not any(fnmatch(target, x) for x in cfg.exclusions)
+            )
+
+        def in_scope(url: str) -> bool:
+            parts = urlsplit(url)
+            return (
+                parts.scheme in ("http", "https")
+                and not parts.username
+                and origin(url) in allowed
+                and not any(fnmatch(url, x) for x in cfg.exclusions)
+            )
+
+        queue = deque((normalize_url(s), 0, True, s) for s in seeds)
+        seen: set[str] = set()
+        dataset = Dataset()
+        pages = total_bytes = 0
+        client = self.client or httpx.Client(
+            timeout=cfg.timeout, follow_redirects=False, trust_env=False
+        )
+        try:
+            while queue and len(seen) < cfg.max_resources:
+                url, depth, page, document = queue.popleft()
+                try:
+                    url = normalize_url(url)
+                except ValueError as exc:
+                    dataset.diagnostics.append(f"Invalid resource URL: {exc}")
+                    continue
+                if url in seen or not in_scope(url) or (page and pages >= cfg.max_pages):
+                    continue
+                seen.add(url)
+                if page:
+                    pages += 1
+                try:
+                    current = url
+                    for redirect in range(cfg.max_redirects + 1):
+                        with client.stream(
+                            "GET",
+                            current,
+                            follow_redirects=False,
+                            timeout=cfg.timeout,
+                            headers={"User-Agent": "SAD/0.1 static-resource-collector"},
+                        ) as response:
+                            if response.is_redirect:
+                                target = normalize_url(
+                                    urljoin(current, response.headers.get("location", ""))
+                                )
+                                if redirect == cfg.max_redirects:
+                                    raise ValueError(f"blocked redirect to {target}")
+                                if not in_scope(target):
+                                    if (
+                                        url in seed_urls
+                                        and canonical_redirect(url, target)
+                                        and canonical_redirect(current, target)
+                                    ):
+                                        allowed.add(origin(target))
+                                    else:
+                                        raise ValueError(f"blocked redirect to {target}")
+                                current = target
+                                continue
+                            response.raise_for_status()
+                            chunks = bytearray()
+                            for chunk in response.iter_bytes():
+                                total_bytes += len(chunk)
+                                if (
+                                    len(chunks) + len(chunk) > cfg.max_bytes
+                                    or total_bytes > cfg.max_total_bytes
+                                ):
+                                    raise ValueError("download byte budget exceeded")
+                                chunks.extend(chunk)
+                            media = response.headers.get("content-type", "").split(";", 1)[0]
+                            resource = Resource(
+                                current,
+                                chunks.decode("utf-8", errors="replace"),
+                                media,
+                                document_url=current if "html" in media else document,
+                            )
+                            dataset.resources.append(resource)
+                            seen.add(current)
+                            break
+                    if total_bytes > cfg.max_total_bytes:
+                        break
+                    if "html" in media:
+                        parser = HTMLCollector(resource)
+                        parser.feed(resource.content)
+                        queue.extend((s, depth, False, parser.base) for s in parser.scripts)
+                        if depth < cfg.depth:
+                            queue.extend((s, depth + 1, True, s) for s in parser.links)
+                    elif (
+                        "javascript" in media
+                        or "ecmascript" in media
+                        or urlsplit(current).path.endswith((".js", ".mjs", ".ts"))
+                    ):
+                        queue.extend(
+                            (s, depth, False, document)
+                            for s in references(resource, self.cache)
+                            if not s.startswith("data:")
+                        )
+                except (httpx.HTTPError, ValueError) as exc:
+                    dataset.diagnostics.append(f"{url}: {exc}")
+                    if total_bytes > cfg.max_total_bytes:
+                        break
+            if queue:
+                dataset.diagnostics.append(
+                    "Collection stopped with queued resources; a configured limit was reached"
+                )
+        finally:
+            if self.client is None:
+                client.close()
+        return dataset

sad/html/__init__.py

@@ -0,0 +1,94 @@
+"""HTML resource and declarative form extraction using the standard HTML parser."""
+
+from __future__ import annotations
+
+from html.parser import HTMLParser
+from urllib.parse import urljoin
+
+from sad.models import Endpoint, Evidence, Resource
+
+
+class HTMLCollector(HTMLParser):
+    def __init__(self, resource: Resource) -> None:
+        super().__init__(convert_charrefs=True)
+        self.resource = resource
+        self.base = resource.url
+        self.links: list[str] = []
+        self.scripts: list[str] = []
+        self.inline: list[Resource] = []
+        self.forms: list[Endpoint] = []
+        self._script: list[str] | None = None
+        self._line = 0
+        self._form: Endpoint | None = None
+        self._base_seen = False
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        values = dict(attrs)
+        line, col = self.getpos()
+        if tag == "base" and values.get("href") and not self._base_seen:
+            self.base = urljoin(self.resource.url, values["href"] or "")
+            self._base_seen = True
+        if tag == "a" and values.get("href"):
+            self.links.append(urljoin(self.base, values["href"] or ""))
+        if tag == "link" and values.get("rel") in ("modulepreload", "preload"):
+            if values.get("as") == "script" or values.get("rel") == "modulepreload":
+                self.scripts.append(urljoin(self.base, values.get("href") or ""))
+        if tag == "script":
+            script_type = (values.get("type") or "").lower()
+            if script_type not in ("", "module", "text/javascript", "application/javascript"):
+                return
+            if values.get("src"):
+                self.scripts.append(urljoin(self.base, values["src"] or ""))
+            else:
+                self._script, self._line = [], line - 1
+        if tag == "form":
+            method = (values.get("method") or "GET").upper()
+            if method == "DIALOG":
+                return
+            self._form = Endpoint(
+                method=method,
+                url=urljoin(self.base, values.get("action") or self.resource.url),
+                source_file=self.resource.source_file or self.resource.url,
+                source_line=line,
+                source_column=col + 1,
+                framework="html-form",
+                content_type=values.get("enctype") or "application/x-www-form-urlencoded",
+                body={} if method != "GET" else None,
+                evidence=[
+                    Evidence(
+                        "html-form",
+                        self.get_starttag_text() or "",
+                        self.resource.url,
+                        line,
+                        col + 1,
+                    )
+                ],
+            )
+            self.forms.append(self._form)
+        if tag in ("input", "textarea", "select", "button") and self._form and values.get("name"):
+            if "disabled" in values:
+                return
+            name = values["name"] or ""
+            value = values.get("value")
+            if self._form.method == "GET":
+                self._form.query_parameters.append({"name": name, "value": value})
+            else:
+                self._form.body[name] = value
+
+    def handle_data(self, data: str) -> None:
+        if self._script is not None:
+            self._script.append(data)
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag == "script" and self._script is not None:
+            self.inline.append(
+                Resource(
+                    self.resource.url + f"#inline-{len(self.inline)}.js",
+                    "".join(self._script),
+                    source_file=self.resource.source_file or self.resource.url,
+                    line_offset=self._line,
+                )
+            )
+            self._script = None
+        if tag == "form":
+            self._form = None