sad-py 0.1.0__tar.gz

sad_py-0.1.0/.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly

sad_py-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: CI
+on: [push, pull_request, workflow_call]
+permissions:
+  contents: read
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.11', '3.12', '3.13', '3.14']
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python }}
+      - run: python -m pip install -e '.[dev]'
+      - run: ruff check src tests examples scripts
+      - run: ruff format --check src tests examples scripts
+      - run: mypy src/sad
+      - run: pytest -m 'not stress' --cov=sad --cov-branch --cov-report=term-missing --cov-report=json:artifacts/coverage.json --cov-report=xml:artifacts/coverage.xml --junitxml=artifacts/tests.xml
+      - run: python scripts/reliability_report.py
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: reliability-python-${{ matrix.python }}
+          path: artifacts/
+      - run: python -m build
+      - run: python -m twine check dist/*
+      - name: Install and smoke-test wheel outside checkout
+        shell: bash
+        run: |
+          python -m venv /tmp/sad-wheel
+          /tmp/sad-wheel/bin/pip install dist/*.whl
+          cd /tmp
+          /tmp/sad-wheel/bin/sad --version
+          /tmp/sad-wheel/bin/python -c 'from sad import Analyzer; assert Analyzer().analyze_js("fetch(\"/smoke\")").endpoints[0].url == "/smoke"'
+  platform-smoke:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, macos-latest]
+        python: ['3.11', '3.14']
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python }}
+      - run: python -m pip install -e '.[dev]'
+      - run: pytest tests/test_ingest.py tests/test_crawler_cli.py tests/test_collection_boundaries.py tests/test_properties.py
+      - run: python -m build
+      - run: python -m pip install --force-reinstall --no-deps --find-links dist sad-py
+      - run: sad --version

sad_py-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+name: Publish to PyPI
+on:
+  release:
+    types: [published]
+permissions:
+  contents: read
+jobs:
+  checks:
+    uses: ./.github/workflows/ci.yml
+  publish:
+    needs: checks
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+      - run: python -m pip install build twine
+      - name: Validate release version and distributions
+        env:
+          RELEASE_TAG: ${{ github.event.release.tag_name }}
+        run: |
+          python -c 'import os,tomllib; assert os.environ["RELEASE_TAG"] == "v" + tomllib.load(open("pyproject.toml","rb"))["project"]["version"]'
+          python -m build
+          python -m twine check dist/*
+      - uses: pypa/gh-action-pypi-publish@release/v1

sad_py-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,34 @@
+name: Draft release
+on:
+  push:
+    tags: ['v*']
+permissions:
+  contents: read
+jobs:
+  checks:
+    uses: ./.github/workflows/ci.yml
+  draft:
+    needs: checks
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+      - run: python -m pip install build twine
+      - name: Verify tag and build
+        env:
+          RELEASE_TAG: ${{ github.ref_name }}
+        run: |
+          python -c 'import os,tomllib; assert os.environ["RELEASE_TAG"] == "v" + tomllib.load(open("pyproject.toml","rb"))["project"]["version"]'
+          python -m build
+          python -m twine check dist/*
+      - name: Create draft with distributions
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_TAG: ${{ github.ref_name }}
+        run: gh release create "$RELEASE_TAG" dist/* --draft --verify-tag --generate-notes

sad_py-0.1.0/.github/workflows/reliability.yml

@@ -0,0 +1,27 @@
+name: Extended reliability
+on:
+  schedule:
+    - cron: '17 3 * * 1'
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  extended:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.14'
+      - run: python -m pip install -e '.[dev]'
+      - run: pytest --cov=sad --cov-branch --cov-report=json:artifacts/coverage.json --junitxml=artifacts/tests.xml
+      - run: python scripts/reliability_report.py
+      - run: python scripts/benchmark.py --operations 10000
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: extended-reliability
+          path: artifacts/

sad_py-0.1.0/.gitignore

@@ -0,0 +1,13 @@
+.venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+dist/
+build/
+*.egg-info/
+artifacts/
+.hypothesis/

sad_py-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+  - repo: local
+    hooks:
+      - id: ruff-check
+        name: ruff check
+        entry: .venv/bin/ruff check --fix
+        language: system
+        types: [python]
+      - id: ruff-format
+        name: ruff format
+        entry: .venv/bin/ruff format
+        language: system
+        types: [python]

sad_py-0.1.0/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+## 0.1.0 — Unreleased
+
+- Library-first Analyzer, public dataclasses and versioned JSON model.
+- Tree-sitter JavaScript, TypeScript, JSX and TSX parsing with bounded AST caching.
+- Symbolic strings, aliases, objects, destructuring, conditional value joins, modules, wrapper chains, closures and basic class clients.
+- HTTP and protocol adapters, HTML forms, programmatic forms, source-map embedded sources and literal lazy imports.
+- Offline Burp XML, normalized JSON, HAR, raw HTTP and directory imports; captured responses feed static analysis.
+- Conservative static/observed correlation with original observations retained.
+- Bounded scoped crawler, CLI and JSON/JSONL/CSV/table serialization.
+- Structured incompleteness issues, isolated branch heaps, return unions, bounded loops/recursive summaries, live module bindings, path mappings and bound calls.
+- Static bundle module tables, indexed/VLQ source maps, client state tracking and Connect wire descriptors.
+- An offline labeled detection corpus, pinned upstream examples, property/fuzz tests, separate line/branch gates and cross-platform CI artifacts.
+- Documented endpoint origins and captured-response discovery in the README.
+- Canonical seed redirects, fair source shares of the global analysis budget, retained partial exports, configurable CLI analysis steps and explicit partial-result status.
+- CI, typed package, wheel/sdist validation, draft releases and trusted PyPI publishing workflow.
+
+This is an initial implementation with the coverage limits in docs/architecture.md. No public release has been made.

sad_py-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,29 @@
+# Contributing
+
+Use Python 3.11 or newer and a virtual environment:
+
+```sh
+python -m venv .venv
+. .venv/bin/activate
+pip install -e '.[dev]'
+ruff format src tests
+ruff check src tests
+mypy src/sad
+pytest -m "not stress" --cov=sad --cov-branch --cov-report=term-missing --cov-report=json:artifacts/coverage.json
+python scripts/reliability_report.py
+pytest -m stress
+python scripts/benchmark.py
+python -m build
+python -m twine check dist/*
+```
+
+Add a minimal, synthetic fixture for each new behavior, plus a false-positive regression when adding sink recognition. Include expected methods, symbolic URLs, provenance and diagnostics. Never commit customer captures, cookies, authorization headers, secrets, or identifying production source maps.
+
+Keep I/O outside the JavaScript value engine. Network sinks require resolved provenance; matching a method name alone is insufficient. Unknown values must survive as symbolic values or explicit nulls. All new syntax handling must honor the analysis budgets and never execute target code.
+
+Public model changes need a changelog entry, schema update, and compatibility assessment. Before 1.0, incompatible changes require a minor version bump; additive compatible fields may ship in patch releases. Keep `Analyzer`, input adapters, and serialization independent of CLI argument parsing.
+
+Run the complete CI matrix before release. Performance changes should include a representative synthetic bundle measurement and preserve deterministic outputs. See docs/architecture.md for limitations that need contributors.
+
+
+Normal tests run offline with real socket connections and child-process execution blocked. Do not add xfail to supported corpus cases or reduce independently enforced 95% line/90% branch gates. Keep research cases outside supported recall totals. Upstream fixtures require pinned commits, content hashes, retained licenses and independently reviewed expected operations.

sad_py-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 SAD contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sad_py-0.1.0/PKG-INFO

@@ -0,0 +1,122 @@
+Metadata-Version: 2.5
+Name: sad-py
+Version: 0.1.0
+Summary: Static, evidence-backed discovery of frontend API operations
+Author: SAD contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: api,burp,javascript,security,static-analysis
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: defusedxml<0.8,>=0.7
+Requires-Dist: httpx<0.29,>=0.28
+Requires-Dist: tree-sitter-javascript<0.26,>=0.23
+Requires-Dist: tree-sitter-typescript<0.24,>=0.23
+Requires-Dist: tree-sitter<0.26,>=0.25
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: hypothesis>=6; extra == 'dev'
+Requires-Dist: jsonschema>=4.23; extra == 'dev'
+Requires-Dist: mypy>=1.15; extra == 'dev'
+Requires-Dist: pytest-cov>=6; extra == 'dev'
+Requires-Dist: pytest-timeout>=2.3; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.11; extra == 'dev'
+Requires-Dist: twine>=6; extra == 'dev'
+Requires-Dist: types-defusedxml; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# SAD: Statistic API Discovery
+
+SAD is a Python library and CLI for evidence-backed static discovery of frontend API operations, including operations absent from captured traffic. Intended for authorized application analysis and security testing. Target JavaScript is never executed.
+
+**Status:** early release, with a bounded abstract interpreter and explicit coverage limits; not a claim of complete JavaScript semantics or exhaustive discovery.
+
+## Install
+
+Requires Python 3.11+. From this repository:
+
+```sh
+pip install .
+# Contributors
+pip install -e '.[dev]'
+```
+
+The PyPI distribution name is `sad-py`; the Python import and executable are `sad`. Once published, install it with `pip install sad-py`. No release has been published by this repository setup.
+
+## Python
+
+```python
+from sad import Analyzer
+from sad.ingest import BurpImporter
+
+analyzer = Analyzer()
+result = analyzer.analyze_url("https://example.com")
+for endpoint in result.endpoints:
+    print(endpoint.method, endpoint.url, endpoint.evidence)
+
+result = analyzer.analyze_dataset(BurpImporter.from_xml("history.xml"))
+result = analyzer.analyze_js('fetch(`/users/${id}`, {method: "DELETE"})')
+result = analyzer.analyze_path("./assets")
+```
+
+## CLI
+
+```sh
+sad crawl https://example.com --depth 2 --max-pages 30
+sad analyze ./assets --format json
+sad analyze-js app.js --base-url https://example.com
+sad import-burp history.xml --format jsonl
+sad import-burp requests.json --format csv
+sad import-burp traffic.har
+sad import-burp request.http --base-url https://example.com
+```
+
+The crawler follows canonical redirects of the starting URL between a host and its `www` variant, and from HTTP to HTTPS on standard ports. It adds the redirected origin to the collection scope so its pages and scripts can be collected. Other origins require explicit scope configuration through `--allow-origin` or `CrawlConfig.allowed_origins`; exclusions and redirect limits still apply. Discovering an API never causes it to be requested.
+
+## Analysis budgets and partial results
+
+`--depth` and `--max-pages` control collection. They do not increase the JavaScript analysis budget. Every command accepts `--max-steps` (default: 200,000), corresponding to `Analyzer(max_steps=...)`:
+
+```sh
+sad crawl https://www.google.com/ --depth 2 --max-pages 30 --max-steps 2000000 --format json --strict
+```
+
+The global step budget is divided among supplied JavaScript sources so one expensive script cannot prevent independent sources from being analyzed. When a source exhausts its share, completed discoveries and export bindings are retained, a located `analysis-budget` issue is emitted and other sources continue. Raising the budget allows more exploration, but does not guarantee completeness or remove other limits.
+
+`Argument combinations exceeded 16` reports widening of too many possible argument combinations. SAD keeps explored alternatives and a symbolic remainder and sets `analysis_incomplete=true`; it does not claim that every concrete combination was analyzed. Loops, recursion and missing inputs can also cause partial results. The CLI explicitly marks incomplete output, and `--strict` exits with status 1 when diagnostics occur.
+
+`high` confidence applies to an individual detected operation, not to coverage of the whole application. An HTML form can produce an endpoint even if JavaScript analysis is incomplete; inspect its `evidence` in JSON to distinguish form declarations from JavaScript sinks.
+
+## Burp history: observed and previously unseen APIs
+
+SAD imports the captured requests **and analyzes captured response bodies**. HTML responses contribute inline scripts and forms; JavaScript responses feed the same static analysis engine used for source files. This can discover operations that were never invoked during the captured session. Assets absent from the capture are not downloaded by an offline import.
+
+For example, a captured `app.js` response might contain:
+
+```javascript
+fetch("/api/users", {method: "POST"});
+fetch(`/api/users/${id}`, {method: "DELETE"});
+fetch("/api/admin/export");
+```
+
+If the history contains `POST /api/users`, SAD correlates that request with the first static discovery. If it also contains `DELETE /api/users/123`, SAD can correlate it with the symbolic `/api/users/{id}`. The export operation remains a static discovery even if no export request appears in the history. Original observations and contributing evidence are retained.
+
+## Endpoint origin states
+
+The `origin` field describes **where the evidence came from**:
+
+| State | Meaning | Example |
+| --- | --- | --- |
+| `static-analysis` | Discovered by reading source code or an HTML declaration, including code inside captured responses. No matching request was supplied. | A captured script calls `fetch("/api/admin/export")`, but that URL is absent from the history. |
+| `observed` | A request exists in the supplied traffic, with no matching static discovery. SAD did not make that request. | Burp recorded `GET /api/profile`, but the supplied code does not expose a corresponding call. |
+| `observed+static` | An observed request was correlated with a static discovery. The logical endpoint retains both evidence sources. | Code declares `DELETE /api/users/{id}` and Burp recorded `DELETE /api/users/123`. |
+| `inferred` | Reserved for future discoveries based on indirect heuristics that cannot yet be traced to a resolved network sink or an observed request. **The current engine does not assign this state automatically.** | A future adapter might suggest a route from incomplete framework metadata. |
+
+`origin` and `confidence` answer different questions: origin identifies the evidence source; confidence expresses certainty in the mapping. An unresolved URL such as `/api/users/{id}` remains `static-analysis` with symbolic values and appropriate confidence. Resolving a wrapper to `fetch` is also static analysis; the word “inference” does not automatically imply `origin="inferred"`.
+
+These states never cause SAD to invoke an endpoint. SAD never executes target JavaScript. Live crawling only collects source resources; importing files or captured traffic is offline.
+
+See [measured reliability and limitations](docs/reliability.md), [architecture and coverage](docs/architecture.md), [integration contract](docs/integrations.md), and [release checklist](docs/releasing.md). JSON, JSONL, CSV and table outputs share the same library models. Diagnostics indicate malformed inputs and exhausted budgets; incomplete results are retained.

sad_py-0.1.0/README.md

@@ -0,0 +1,92 @@
+# SAD: Statistic API Discovery
+
+SAD is a Python library and CLI for evidence-backed static discovery of frontend API operations, including operations absent from captured traffic. Intended for authorized application analysis and security testing. Target JavaScript is never executed.
+
+**Status:** early release, with a bounded abstract interpreter and explicit coverage limits; not a claim of complete JavaScript semantics or exhaustive discovery.
+
+## Install
+
+Requires Python 3.11+. From this repository:
+
+```sh
+pip install .
+# Contributors
+pip install -e '.[dev]'
+```
+
+The PyPI distribution name is `sad-py`; the Python import and executable are `sad`. Once published, install it with `pip install sad-py`. No release has been published by this repository setup.
+
+## Python
+
+```python
+from sad import Analyzer
+from sad.ingest import BurpImporter
+
+analyzer = Analyzer()
+result = analyzer.analyze_url("https://example.com")
+for endpoint in result.endpoints:
+    print(endpoint.method, endpoint.url, endpoint.evidence)
+
+result = analyzer.analyze_dataset(BurpImporter.from_xml("history.xml"))
+result = analyzer.analyze_js('fetch(`/users/${id}`, {method: "DELETE"})')
+result = analyzer.analyze_path("./assets")
+```
+
+## CLI
+
+```sh
+sad crawl https://example.com --depth 2 --max-pages 30
+sad analyze ./assets --format json
+sad analyze-js app.js --base-url https://example.com
+sad import-burp history.xml --format jsonl
+sad import-burp requests.json --format csv
+sad import-burp traffic.har
+sad import-burp request.http --base-url https://example.com
+```
+
+The crawler follows canonical redirects of the starting URL between a host and its `www` variant, and from HTTP to HTTPS on standard ports. It adds the redirected origin to the collection scope so its pages and scripts can be collected. Other origins require explicit scope configuration through `--allow-origin` or `CrawlConfig.allowed_origins`; exclusions and redirect limits still apply. Discovering an API never causes it to be requested.
+
+## Analysis budgets and partial results
+
+`--depth` and `--max-pages` control collection. They do not increase the JavaScript analysis budget. Every command accepts `--max-steps` (default: 200,000), corresponding to `Analyzer(max_steps=...)`:
+
+```sh
+sad crawl https://www.google.com/ --depth 2 --max-pages 30 --max-steps 2000000 --format json --strict
+```
+
+The global step budget is divided among supplied JavaScript sources so one expensive script cannot prevent independent sources from being analyzed. When a source exhausts its share, completed discoveries and export bindings are retained, a located `analysis-budget` issue is emitted and other sources continue. Raising the budget allows more exploration, but does not guarantee completeness or remove other limits.
+
+`Argument combinations exceeded 16` reports widening of too many possible argument combinations. SAD keeps explored alternatives and a symbolic remainder and sets `analysis_incomplete=true`; it does not claim that every concrete combination was analyzed. Loops, recursion and missing inputs can also cause partial results. The CLI explicitly marks incomplete output, and `--strict` exits with status 1 when diagnostics occur.
+
+`high` confidence applies to an individual detected operation, not to coverage of the whole application. An HTML form can produce an endpoint even if JavaScript analysis is incomplete; inspect its `evidence` in JSON to distinguish form declarations from JavaScript sinks.
+
+## Burp history: observed and previously unseen APIs
+
+SAD imports the captured requests **and analyzes captured response bodies**. HTML responses contribute inline scripts and forms; JavaScript responses feed the same static analysis engine used for source files. This can discover operations that were never invoked during the captured session. Assets absent from the capture are not downloaded by an offline import.
+
+For example, a captured `app.js` response might contain:
+
+```javascript
+fetch("/api/users", {method: "POST"});
+fetch(`/api/users/${id}`, {method: "DELETE"});
+fetch("/api/admin/export");
+```
+
+If the history contains `POST /api/users`, SAD correlates that request with the first static discovery. If it also contains `DELETE /api/users/123`, SAD can correlate it with the symbolic `/api/users/{id}`. The export operation remains a static discovery even if no export request appears in the history. Original observations and contributing evidence are retained.
+
+## Endpoint origin states
+
+The `origin` field describes **where the evidence came from**:
+
+| State | Meaning | Example |
+| --- | --- | --- |
+| `static-analysis` | Discovered by reading source code or an HTML declaration, including code inside captured responses. No matching request was supplied. | A captured script calls `fetch("/api/admin/export")`, but that URL is absent from the history. |
+| `observed` | A request exists in the supplied traffic, with no matching static discovery. SAD did not make that request. | Burp recorded `GET /api/profile`, but the supplied code does not expose a corresponding call. |
+| `observed+static` | An observed request was correlated with a static discovery. The logical endpoint retains both evidence sources. | Code declares `DELETE /api/users/{id}` and Burp recorded `DELETE /api/users/123`. |
+| `inferred` | Reserved for future discoveries based on indirect heuristics that cannot yet be traced to a resolved network sink or an observed request. **The current engine does not assign this state automatically.** | A future adapter might suggest a route from incomplete framework metadata. |
+
+`origin` and `confidence` answer different questions: origin identifies the evidence source; confidence expresses certainty in the mapping. An unresolved URL such as `/api/users/{id}` remains `static-analysis` with symbolic values and appropriate confidence. Resolving a wrapper to `fetch` is also static analysis; the word “inference” does not automatically imply `origin="inferred"`.
+
+These states never cause SAD to invoke an endpoint. SAD never executes target JavaScript. Live crawling only collects source resources; importing files or captured traffic is offline.
+
+See [measured reliability and limitations](docs/reliability.md), [architecture and coverage](docs/architecture.md), [integration contract](docs/integrations.md), and [release checklist](docs/releasing.md). JSON, JSONL, CSV and table outputs share the same library models. Diagnostics indicate malformed inputs and exhausted budgets; incomplete results are retained.

sad_py-0.1.0/SECURITY.md

@@ -0,0 +1,9 @@
+# Security policy
+
+SAD analyzes untrusted code without executing it. Report vulnerabilities through the repository's GitHub **Security → Report a vulnerability** facility once private reporting is enabled. If unavailable, open an issue requesting a private contact without exploit details or sensitive data. There is no staffed response SLA yet. Security fixes target the latest release.
+
+The AST engine has byte, step, recursion and cache limits. XML imports use defusedxml. Source-map source names are virtual identifiers and never authorize local reads. Crawler redirects are scope-checked, TLS verification stays enabled, and ambient proxy credentials are disabled. Imported captures and local analysis do not trigger network access.
+
+The crawler intentionally supports internal targets for authorized assessments. Its same-origin policy is not an SSRF defense for an internet-facing service: DNS rebinding and access to explicitly seeded private hosts are not blocked. Integrators accepting untrusted crawl seeds must enforce network egress policy outside SAD. Run hostile workloads in a resource-limited worker; native parsers and bounded interpretation are not a process isolation boundary.
+
+Output preserves request headers and bodies, which can include credentials and personal information. Protect output as sensitive assessment data. Raw HTTP import expects decoded, dechunked bodies; it does not transparently decompress captures.

sad_py-0.1.0/docs/architecture.md

@@ -0,0 +1,101 @@
+# Architecture and coverage
+
+SAD is a bounded static analyzer, not a JavaScript runtime. Its success measure is evidence-backed coverage of potentially invokable operations. Results are candidates, not proof of reachability, successful authorization, or server existence.
+
+```mermaid
+flowchart LR
+    A[Files / captured traffic / scoped crawler] --> B[Dataset]
+    B --> C[HTML and source-map expansion]
+    C --> D[Cached JS/TS ASTs]
+    D --> E[Modules and abstract value propagation]
+    E --> F[Resolved network sinks]
+    F --> G[Normalization and correlation]
+    B --> G
+    G --> H[Python / JSON / JSONL / CSV / CLI]
+```
+
+## Module responsibilities
+
+| Module | Responsibility |
+| --- | --- |
+| `models` | Public input, evidence, endpoint and result dataclasses |
+| `ingest` | Capture adapters and bounded input reads |
+| `html` | HTMLParser-based scripts, links and forms |
+| `javascript/parser` | Tree-sitter grammars and locked content-addressed LRU |
+| `javascript/values` | Unknown, function closure, client, bound member and value unions |
+| `javascript/engine` | Module graph, environments, calls, bounded interpretation and sink adapters |
+| `javascript/resources` | AST import discovery and source-map embedded sources |
+| `protocols` | Extensible package/global sink registry |
+| `crawler` | Scoped resource acquisition; never invokes discovered APIs |
+| `normalization` | Canonicalization, exact/symbolic correlation and deduplication |
+| `output`, `cli` | Frontends over the same result objects |
+
+The engine loads available modules on demand, links imports, resolves constants/objects/member accesses, and propagates abstract arguments through calls. Calls record caller/callee edges. Uncalled function bodies are explored with symbolic parameters to discover hidden APIs. This can emit both a broad wrapper candidate and a specialized caller result. Branches are explored independently and simple scalar assignments are joined into bounded alternatives. Aliased known clients remain sinks; an arbitrary object method called `get` does not.
+
+## Current coverage
+
+| Input or pattern | Implemented behavior | Limits |
+| --- | --- | --- |
+| JS, TS, JSX, TSX, minified code | Tree-sitter parsing and error recovery | Unsupported syntax is traversed for nested sinks; not full semantics |
+| Strings and configuration | Constants, concatenation, templates, objects, spread, explicit environment values | No arbitrary built-in execution or computed string algorithms |
+| Calls | Function declarations, arrows, aliases, destructuring, nested wrappers, returned closures, basic classes | Basic inheritance, receivers and `bind`/`call`/`apply` are supported; decorators, prototype mutation and a full async scheduling model remain incomplete |
+| Branches | Isolated abstract branch heaps, return unions, aliases, bounded loops and recursive summaries | Up to 16 alternatives, 64 loop iterations and eight recursive-summary rounds; widening and recursion emit issues. Exception paths are conservative. |
+| Modules | Relative ESM/CommonJS, default/named exports, re-exports, namespace imports, literal dynamic imports | Cyclic binding cells and explicitly supplied path mappings are supported; package exports and automatic tsconfig loading remain incomplete |
+| fetch, XHR, beacon | URLs, methods, request objects, headers and bodies | XHR open/send/header mutations are linked; response semantics remain incomplete |
+| Axios, jQuery | Calls, HTTP verbs, factory base URLs and headers | Factory/default configuration and request overrides supported; interceptors remain incomplete |
+| ky, got, ofetch, unfetch, request, superagent, wretch | Imported client conventions, common factories, verb calls; basic wretch chains | Fluent response/body modifiers and advanced per-library options incomplete |
+| Angular | Imported HttpClient parameter annotation recognized | Framework DI/decorators and constructor property promotion incomplete |
+| Custom/React/Vue/Svelte/generated clients | Follow their source to known sinks; explicit package registration | Opaque external implementations need adapters; no framework-name guessing |
+| GraphQL/Apollo/urql/graphql-request | Configured URLs and common query/request calls; JSON query payload detection | Operation parsing, Apollo link composition, Relay artifacts/subscriptions incomplete |
+| WebSocket, Socket.IO, SSE | Connection targets from constructors/imported clients | Event channels, reconnection configuration, Socket.IO transport path not modeled |
+| JSON-RPC, SOAP | Structured JSON-RPC bodies and SOAP content-type/header evidence | No WSDL/service discovery |
+| gRPC-Web | `GrpcWebClientBase` RPC URL calls and content-type evidence | Protobuf reflection and opaque generated modules incomplete |
+| Connect-Web, tRPC | Known transport/client factories and conventional method/procedure paths | Literal Connect descriptors supply wire names; opaque protobuf encodings, batching and advanced link chains remain incomplete |
+| Forms | Declarative forms, controls, basic `createElement('form')` action/method/submit | No DOM tree execution, event-driven mutation, form-owner lookup or shadow DOM |
+| Bundles/chunks | Parse ordinary JS in bundles, inspect uncalled factories, follow literal imports | Static numeric Webpack/Parcel module tables and literal ESM chunks are supported; arbitrary loader execution and computed manifests remain unsupported |
+| Source maps | External maps supplied/collected, base64 inline maps, embedded sources with original names/lines | VLQ original locations and inline indexed sections are supported; external indexed sections must be supplied, and source names never authorize filesystem reads |
+
+## Confidence and evidence
+
+Every endpoint has AST/form/capture evidence and a location when available. Calls through wrappers include their call chain; configured bases include factory evidence. Source-map embedded sources use `map#section-N:source-N:original-name` provenance.
+
+- **High:** a resolved known network sink and concrete URL; observed traffic and HTML declarations also qualify. A high score does not prove runtime reachability.
+- **Medium:** symbolic URL segments, or conventional tRPC/Connect paths whose wire representation may differ.
+- **Low:** a URL beginning with an unresolved symbolic base.
+
+Unknown metadata uses JSON null, and symbolic strings use `{id}` or `{dynamic}`. Unknown methods are null, never silently asserted as GET. GET is used only for APIs with that default. `origin` is `static-analysis`, `observed`, `observed+static`, or reserved `inferred`; current protocol heuristics use static-analysis with medium confidence.
+
+## Correlation
+
+Canonicalization lowercases scheme/host, removes fragments and default ports, and preserves path case, encoded delimiters and query order. Correlation requires matching protocol, origin and method, preferring exact paths over symbolic paths. Queries are retained in parameters and observations. Multiple different numeric observed paths with a shared structure can be grouped under `{id}`; a single numeric path is kept concrete. This is a heuristic, not a claim of server routing semantics. Original URLs, headers, bodies, status and evidence remain attached.
+
+A logical endpoint's top-level request metadata is a representative, not a complete union of every possible static payload. Evidence retains the contributing calls; raw traffic variants remain in observations and merged static configurations in request_variants. Multiple GraphQL operations sharing a transport URL are not yet modeled as separate operations.
+
+## Budgets, determinism and concurrency
+
+Default AST cache: 128 entries, keyed by grammar and SHA-256 content, with an 8 MB per-resource limit. Default interpreter budgets: 200,000 steps, call depth 20, 16 branch alternatives, 64 loop iterations and eight recursive-summary rounds. Reuse an Analyzer to avoid reparsing unchanged content; value evaluation remains per analysis. The cache lock protects parsing, but parallel parsing is not enabled. For parallel analyses, use separate Analyzer instances per worker rather than modifying a registry concurrently.
+
+The step limit is a global bound, divided into source shares after parsing. Each source's top-level code and function exploration consume its share; imported code consumes the defining source's share. Exhausting a source retains its partial exports and operations and emits an `analysis-budget` issue with that source's location. Independent sources continue. Unused shares are not reassigned, and a budget too small to visit every source still stops at the global bound. The CLI exposes `--max-steps`; `--depth` and `--max-pages` govern collection rather than abstract interpretation. Increasing steps does not change widening, recursion or loop limits.
+
+Crawling defaults to depth 2, 30 pages, 300 resources, 8 MB per response, 64 MB aggregate decoded bytes, a 10-second request timeout, and five redirects. Diagnostics identify truncation and unavailable imports. Inputs are analyzed in deterministic order and results are sorted. This is not a whole-project incremental data-flow database.
+
+Crawler resources preserve their document URL for relative fetch semantics. Offline captured JS uses its asset URL when neither `Resource.document_url` nor `base_url` supplies document context; provide that context when it matters. An explicit base applies across the analysis. Local source-map names are never read from disk automatically.
+
+## Next substantial milestones
+
+1. A complete control-flow graph and stronger exception/prototype semantics beyond the bounded structured analysis.
+2. Declarative, independently testable sink adapters and complete generated protocol descriptors.
+3. Computed bundle/chunk manifests, complete client mutation tracking and opaque generated protocol descriptors.
+4. Broader real-world corpora, persistent incremental summaries and safely parallel parsing.
+5. Native Burp Montoya companion, per-operation GraphQL/RPC identities, OpenAPI export, diffs and issue generation.
+
+These are outstanding engineering work, not advertised as implemented behavior.
+
+
+## Structured incompleteness
+
+`AnalysisResult.issues` contains structured codes, reasons and source locations. `analysis_incomplete` is true when parsing recovered errors, imports or wire descriptors are absent, URL/method components remain symbolic, or a resource/value/loop/recursion budget was reached. Existing string diagnostics remain available. No issues does not establish exhaustive JavaScript coverage.
+
+`Analyzer(path_mappings={"@api/*": "/lib/*"})` resolves aliases exclusively against supplied resources. `Resource.document_url` supplies document context without any network access. JSON schema 1.0 keeps the new result fields optional for validation of older outputs; new SAD results always include them.
+
+The [reliability report](reliability.md) documents the labeled corpus, measured code coverage, property tests and retained research cases.