pydj 0.1.0__tar.gz

pydj-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,107 @@
+name: Bug report
+description: Something is broken — a crash, wrong output, or a site not extracted correctly
+title: "[Bug]: "
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for the report. The single most useful thing you can include is
+        **the URL**, if a specific site is involved.
+
+  - type: dropdown
+    id: interface
+    attributes:
+      label: How are you using pydj?
+      options:
+        - CLI (pydj / dj)
+        - Library (from pydj import scan)
+        - Both
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: The actual behaviour, including any error output.
+      placeholder: |
+        Running `pydj https://example.com` reports 0 JS files, but the page
+        clearly loads several.
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: What did you expect?
+      placeholder: It should find the scripts referenced by the page.
+    validations:
+      required: true
+
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: Steps to reproduce
+      description: The exact command or code, so it can be run verbatim.
+      render: shell
+      placeholder: |
+        pipx install 'pydj[tls]'
+        pydj -f json --no-cache https://example.com
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: pydj version
+      description: Output of `pydj --version`.
+      placeholder: pydj 0.1.0
+    validations:
+      required: true
+
+  - type: input
+    id: python
+    attributes:
+      label: Python version and OS
+      description: Output of `python -V`, plus your platform.
+      placeholder: Python 3.12.4 on macOS 15.1 (arm64)
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: tls
+    attributes:
+      label: TLS impersonation
+      description: |
+        Whether browser TLS fingerprinting is active changes the outcome on
+        sites behind Cloudflare and similar.
+      options:
+        - label: I installed the `tls` extra (so `curl-cffi` is present)
+        - label: "`pydj --debug` reports TLS fingerprint impersonation is disabled"
+
+  - type: dropdown
+    id: go-comparison
+    attributes:
+      label: Does the Go implementation behave the same way?
+      description: |
+        pydj is a port, so this instantly distinguishes "the port is wrong" from
+        "the original does this too". Skip if you have not tried it.
+      options:
+        - "Not tried"
+        - "Go finds it correctly — pydj is missing it"
+        - "Go behaves the same way"
+        - "Go misses it too"
+    validations:
+      required: true
+
+  - type: textarea
+    id: debug
+    attributes:
+      label: Debug output
+      description: |
+        If you can, attach the tail of `pydj --debug -f json <url>` (it is
+        verbose; the last ~100 lines usually suffice).
+      render: shell
+    validations:
+      required: false

pydj-0.1.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Security vulnerability
+    url: https://github.com/ejfkdev/pydj/security/advisories/new
+    about: Report privately, not as a public issue. See SECURITY.md.
+  - name: Question or usage help
+    url: https://github.com/ejfkdev/pydj/discussions
+    about: For "how do I…" questions. The README and examples/ answer most of them.
+  - name: The original Go implementation
+    url: https://github.com/ejfkdev/dj/issues
+    about: If the Go tool behaves the same way, the issue may belong upstream.

pydj-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,68 @@
+name: Feature request
+description: Suggest a bundler, framework or loading pattern pydj should recognise
+title: "[Feature]: "
+labels: ["enhancement"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Most feature requests here are about **discovery coverage** — a bundler
+        whose chunk URL scheme pydj does not recognise. Those are the easiest to
+        action, and a URL plus the relevant code snippet usually makes it a
+        small, well-defined change.
+
+  - type: dropdown
+    id: kind
+    attributes:
+      label: What kind of request is this?
+      options:
+        - A site or bundler whose JS is not extracted (coverage)
+        - A new library API or option
+        - A new CLI option
+        - Documentation
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: What problem are you hitting?
+      description: Describe the situation, not the solution.
+      placeholder: |
+        A site built with X loads its chunks via `X.load("name")` and pydj finds
+        only the entry bundle.
+    validations:
+      required: true
+
+  - type: textarea
+    id: evidence
+    attributes:
+      label: Evidence from the site
+      description: |
+        If this is a coverage request, the URL and the code that builds the
+        chunk URLs. Grep the bundle for the chunk path pattern — the surrounding
+        function is what a plugin has to match.
+      render: shell
+      placeholder: |
+        # from the site's bundle
+        l.u = function (e) { return "static/js/" + e + "." + {10: "abc123"}[e] + ".js" }
+    validations:
+      required: false
+
+  - type: textarea
+    id: proposal
+    attributes:
+      label: What would you like to see?
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: contribute
+    attributes:
+      label: Would you like to work on it?
+      options:
+        - label: I am willing to open a pull request
+          required: false
+        - label: I have read CONTRIBUTING.md
+          required: false

pydj-0.1.0/.github/pull_request_template.md

@@ -0,0 +1,27 @@
+## What this changes
+
+<!-- One or two sentences. What was wrong or missing, and what the change does. -->
+
+## Why
+
+<!-- The reason, especially if the change looks like it does the wrong thing.
+     Behaviour that deliberately differs from the Go original needs a note here,
+     or a future reader will "fix" it back. -->
+
+## How it was verified
+
+<!-- Delete what does not apply. -->
+
+- [ ] `ruff check src tests examples` passes
+- [ ] `python -m pytest` passes
+- [ ] Checked against the Go implementation on a real site
+      (`DJ_REFERENCE_BIN=/tmp/dj-ref python -m pytest tests/test_differential.py`)
+- [ ] New or changed behaviour has a test
+
+## Notes for review
+
+<!-- Anything you want a reviewer to look at first, or are unsure about.
+
+     If this changes discovery, say whether it can *lose* a URL the previous
+     version found -- a port that silently stops finding a file is worse than one
+     that wastes a request. -->

pydj-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,263 @@
+name: CI
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Test (Python ${{ matrix.python-version }} on ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # 3.10 is the floor declared in pyproject.toml; the rest guard against
+        # regressions on newer interpreters (a decompression bug once appeared
+        # only on 3.14).
+        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
+        os: [ubuntu-latest]
+        include:
+          - { os: macos-latest, python-version: '3.13' }
+          - { os: windows-latest, python-version: '3.13' }
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e '.[dev]'
+
+      - name: Show resolved HTTP stack
+        # curl-cffi is what enables TLS impersonation; knowing which version a
+        # failing run used makes a platform-specific failure much easier to read.
+        run: |
+          python -c "import curl_cffi, httpx, sys; print('python', sys.version.split()[0]); print('curl_cffi', curl_cffi.__version__); print('httpx', httpx.__version__)"
+
+      - name: Test
+        run: python -m pytest -q
+
+  verify-cli:
+    name: Verify CLI end to end
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e '.[dev]'
+
+      - name: Build a fixture site and scan it
+        # The unit tests cover the pipeline in-process; this checks the packaged
+        # entry point actually finds JS over real HTTP, which is the thing users
+        # invoke. Catches anything that only breaks in the console script.
+        run: |
+          python - <<'PY'
+          import json, pathlib, threading, subprocess, sys
+          from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+          root = pathlib.Path('fixture_site')
+          (root / 'js').mkdir(parents=True, exist_ok=True)
+          (root / 'index.html').write_text(
+              '<!doctype html><html><head>'
+              '<link rel="modulepreload" href="/js/vendor.js">'
+              '</head><body>'
+              '<script src="/js/app.js"></script>'
+              '<script>import("/js/lazy.js")</script>'
+              '</body></html>'
+          )
+          (root / 'js/app.js').write_text(
+              'import("./chunk.js");\n//# sourceMappingURL=app.js.map\n'
+          )
+          for name in ('vendor.js', 'lazy.js', 'chunk.js'):
+              (root / 'js' / name).write_text('var x = 1;\n')
+          (root / 'js/app.js.map').write_text(json.dumps({
+              'version': 3, 'sources': ['src/App.tsx'],
+              'sourcesContent': ['export const App = 1;\n'], 'mappings': 'AAAA',
+          }))
+
+          class Handler(BaseHTTPRequestHandler):
+              def do_GET(self):
+                  target = root / self.path.lstrip('/')
+                  if target.is_dir():
+                      target = target / 'index.html'
+                  if not target.is_file():
+                      self.send_response(404); self.send_header('Content-Length','0'); self.end_headers(); return
+                  body = target.read_bytes()
+                  ctype = 'application/json' if target.suffix == '.map' else (
+                      'text/html' if target.suffix == '.html' else 'application/javascript')
+                  self.send_response(200)
+                  self.send_header('Content-Type', ctype)
+                  self.send_header('Content-Length', str(len(body)))
+                  self.end_headers()
+                  self.wfile.write(body)
+              def do_HEAD(self):
+                  self.do_GET()
+              def log_message(self, *a): pass
+
+          server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
+          threading.Thread(target=server.serve_forever, daemon=True).start()
+          url = f'http://127.0.0.1:{server.server_port}/'
+
+          proc = subprocess.run(
+              [sys.executable, '-m', 'pydj', '-f', 'json', '--no-cache', url],
+              capture_output=True, text=True, timeout=180,
+          )
+          server.shutdown(); server.server_close()
+
+          if proc.returncode != 0:
+              sys.exit(f'pydj exited {proc.returncode}\n{proc.stderr}')
+
+          data = json.loads(proc.stdout)
+          found = {u.rsplit('/', 1)[-1] for u in data['jsURLs']}
+          expected = {'app.js', 'vendor.js', 'lazy.js', 'chunk.js'}
+          missing = expected - found
+          if missing:
+              sys.exit(f'CLI missed {sorted(missing)}; found {sorted(found)}')
+
+          summary = data['summary']
+          assert summary['jsCount'] == 4, summary
+          assert summary.get('sourceMapCount') == 1, summary
+          assert summary.get('sourceCount') == 1, summary
+          print('CLI found all four JS files, the source map, and restored 1 source')
+          PY
+
+  differential:
+    name: Differential test against the Go implementation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - uses: actions/setup-go@v5
+        with:
+          # The Go project declares 1.26; `stable` keeps this working when the
+          # action's cache has not seen that version yet.
+          go-version: 'stable'
+
+      - name: Build the reference implementation
+        # This is the oracle the port is measured against. The suite skips
+        # cleanly without it, which is why it needs its own job -- otherwise the
+        # strongest test in the repo never actually runs in CI.
+        #
+        # `continue-on-error` because this job depends on an external repository
+        # and Go toolchain: if the clone fails, the upstream module moves, or the
+        # Go version policy changes, that is not a defect in pydj and must not
+        # fail the build. The next step reports clearly whether the suite
+        # actually ran.
+        continue-on-error: true
+        run: |
+          git clone --depth 1 https://github.com/ejfkdev/dj /tmp/dj-src
+          cd /tmp/dj-src
+          go build -o /tmp/dj-ref .
+          /tmp/dj-ref -h | head -3
+
+      - name: Confirm the oracle is usable
+        # Without this, a failed clone would silently turn the differential suite
+        # into a no-op that still reports success -- the one outcome worse than
+        # not running it.
+        run: |
+          if [ ! -x /tmp/dj-ref ]; then
+            echo "::warning::reference binary unavailable; differential suite will skip"
+            exit 0
+          fi
+          python -c "import subprocess; out = subprocess.run(['/tmp/dj-ref','-h'], capture_output=True, text=True).stdout; assert 'Dynamic JS File Extractor' in out, out[:200]; print('oracle ok')"
+
+      - name: Install pydj
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e '.[dev]'
+
+      - name: Run the differential suite
+        env:
+          DJ_REFERENCE_BIN: /tmp/dj-ref
+        run: python -m pytest tests/test_differential.py -v
+
+      - name: Fail if the suite silently skipped
+        # A skipped differential run looks identical to a passing one in the
+        # summary line, which is the failure mode this guards against.
+        run: |
+          if [ ! -x /tmp/dj-ref ]; then
+            echo "Differential suite did not run: no reference binary."
+            echo "This job is advisory; see the warning above."
+            exit 0
+          fi
+          python -m pytest tests/test_differential.py -q --no-header 2>&1 | tail -3
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e '.[dev]' ruff
+
+      - name: Ruff
+        run: ruff check src tests
+
+      - name: Check the wheel builds and has the expected metadata
+        run: |
+          pip install build
+          python -m build --wheel
+          python - <<'PY'
+          import glob, zipfile
+          wheel = glob.glob('dist/*.whl')[0]
+          archive = zipfile.ZipFile(wheel)
+          # Locate the dist-info directory rather than reconstructing its name:
+          # the wheel filename carries build tags (py3-none-any) that the
+          # dist-info directory does not.
+          dist_info = next(n for n in archive.namelist() if n.endswith('.dist-info/METADATA'))
+          meta = archive.read(dist_info).decode()
+
+          # curl-cffi must be an unconditional dependency: without it, TLS
+          # impersonation silently stops working, which is the whole product.
+          assert 'Requires-Dist: curl-cffi' in meta, 'curl-cffi must be a hard dependency'
+          assert 'Requires-Dist: httpx' in meta
+          assert 'Homepage, https://github.com/ejfkdev/pydj' in meta, 'wrong project URL'
+          assert 'Project-URL: Repository, https://github.com/ejfkdev/pydj' in meta
+          # PEP 639: the license is reported as an expression, not a free-text
+          # classifier. Asserting the old `License:` field would fail here even
+          # though the metadata is correct.
+          assert 'License-Expression: MPL-2.0' in meta, 'wrong license metadata'
+          assert 'License-File: LICENSE' in meta
+
+          entry_points = archive.read(
+              dist_info.replace('METADATA', 'entry_points.txt')
+          ).decode()
+          assert 'pydj = pydj.cli:main' in entry_points, 'pydj console script missing'
+          assert 'dj = pydj.cli:main' in entry_points, 'dj console script missing'
+
+          # The README is the PyPI description; an empty one means a packaging
+          # regression that only shows up on the project page.
+          assert 'Dynamic JS File Extractor' in meta, 'long_description missing'
+
+          print('wheel metadata OK:', dist_info.split('/')[0])
+          PY

pydj-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,146 @@
+name: Release
+
+# Tag-driven, mirroring the Go project's build.yml: pushing `v0.2.0` is the only
+# action required to cut a release.
+on:
+  push:
+    tags:
+      - 'v*'
+
+permissions:
+  contents: read
+
+jobs:
+  # Gate the release on the same checks CI runs. A tag that does not pass tests
+  # never reaches PyPI, where a bad upload can only be yanked, not deleted.
+  verify:
+    name: Verify before publishing
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e '.[dev]'
+
+      - name: Test
+        run: python -m pytest -q
+
+      - name: Check the tag matches the packaged version
+        # A mismatch would publish a wheel whose version disagrees with its tag,
+        # and PyPI refuses re-uploads, so catch it here.
+        run: |
+          python - <<'PY'
+          import os, re, pathlib
+          tag = os.environ['GITHUB_REF_NAME'].lstrip('v')
+          text = pathlib.Path('pyproject.toml').read_text()
+          version = re.search(r'^version = "([^"]+)"', text, re.M).group(1)
+          init = pathlib.Path('src/pydj/__init__.py').read_text()
+          init_version = re.search(r'__version__ = "([^"]+)"', init).group(1)
+          if not (tag == version == init_version):
+              raise SystemExit(
+                  f'version mismatch: tag={tag} pyproject={version} __init__={init_version}'
+              )
+          print(f'version {version} consistent across tag, pyproject.toml and __init__.py')
+          PY
+
+  build:
+    name: Build distributions
+    needs: verify
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Build sdist and wheel
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+
+      - name: Inspect the artefacts
+        run: |
+          python -m pip install twine
+          twine check dist/*
+          ls -la dist/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+          retention-days: 7
+
+  pypi:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    # Trusted publishing (OIDC) -- no long-lived token in the repository. Set
+    # this up once at https://pypi.org/manage/account/publishing/ with:
+    #   owner: ejfkdev   repo: pydj
+    #   workflow: release.yml   environment: pypi
+    # `id-token: write` is what mints the short-lived credential.
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pydj
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish
+        # Falls back to a PYPI_API_TOKEN secret if one is configured, so the
+        # workflow works whether or not trusted publishing has been set up yet.
+        # Prefer OIDC: nothing to rotate, and a leaked repo cannot publish.
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN || '' }}
+          # Nothing to attach the attestations to otherwise.
+          attestations: ${{ secrets.PYPI_API_TOKEN == '' }}
+
+  github-release:
+    name: Create GitHub Release
+    needs: pypi
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      # On a tag push, checkout overwrites the local tag ref with a bare commit
+      # SHA; re-fetching the annotated tag lets --notes-from-tag read its message.
+      - name: Fetch tag annotation
+        run: git fetch --force origin "+refs/tags/${{ github.ref_name }}:refs/tags/${{ github.ref_name }}"
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Create release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release create "${{ github.ref_name }}" dist/* \
+            --title "${{ github.ref_name }}" \
+            --notes-from-tag \
+            --verify-tag
+
+      - name: Announce the install command
+        run: |
+          echo "Published. Install with:" >> "$GITHUB_STEP_SUMMARY"
+          echo '```bash' >> "$GITHUB_STEP_SUMMARY"
+          echo "pipx install pydj" >> "$GITHUB_STEP_SUMMARY"
+          echo "uvx pydj --help" >> "$GITHUB_STEP_SUMMARY"
+          echo '```' >> "$GITHUB_STEP_SUMMARY"

pydj-0.1.0/.gitignore

@@ -0,0 +1,70 @@
+# Byte-compiled / cache
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+pip-wheel-metadata/
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+env.bak/
+venv.bak/
+
+# Test / coverage artefacts
+.pytest_cache/
+.tox/
+.nox/
+.coverage
+.coverage.*
+coverage.xml
+*.cover
+.hypothesis/
+htmlcov/
+
+# Type checkers / linters
+.mypy_cache/
+.dmypy.json
+.pyre/
+.pytype/
+.ruff_cache/
+
+# Build tool caches
+.uv/
+
+# Editors / OS
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+.DS_Store
+Thumbs.db
+
+# Generated by examples/06_batch_scan.py
+batch_results.json
+
+# A Go checkout used as the differential oracle
+/dj-ref