crawlee 0.6.12b29__tar.gz → 1.0.5b21__tar.gz

crawlee-1.0.5b21/.github/workflows/build_and_deploy_docs.yaml

@@ -0,0 +1,72 @@
+name: Build and deploy docs
+
+on:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      ref:
+        required: true
+        type: string
+
+env:
+  NODE_VERSION: 20
+  PYTHON_VERSION: 3.14
+
+jobs:
+  build_and_deploy_docs:
+    environment:
+      name: github-pages
+    permissions:
+      contents: write
+      pages: write
+      id-token: write
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+        with:
+          token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+          ref: ${{ github.event_name == 'workflow_call' && inputs.ref || github.ref }}
+
+      - name: Set up Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Set up uv package manager
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Python dependencies
+        run: make install-dev
+
+      - name: Build generated API reference
+        run: make build-api-reference
+
+      - name: Build Docusaurus docs
+        run: make build-docs
+        env:
+          APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
+
+      - name: Set up GitHub Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload GitHub Pages artifact
+        uses: actions/upload-pages-artifact@v4
+        with:
+          path: ./website/build
+
+      - name: Deploy artifact to GitHub Pages
+        uses: actions/deploy-pages@v4
+
+      - name: Invalidate CloudFront cache
+        run: gh workflow run invalidate.yaml --repo apify/apify-docs-private
+        env:
+          GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}

crawlee-1.0.5b21/.github/workflows/check_pr_title.yaml

@@ -0,0 +1,14 @@
+name: Check PR title
+
+on:
+  pull_request_target:
+    types: [opened, edited, synchronize]
+
+jobs:
+  check_pr_title:
+    name: Check PR title
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v6.1.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

crawlee-1.0.5b21/.github/workflows/pre_release.yaml

@@ -0,0 +1,88 @@
+name: Create a pre-release
+
+on:
+  # Trigger a beta version release (pre-release) on push to the master branch.
+  push:
+    branches:
+      - master
+    tags-ignore:
+      - "**" # Ignore all tags to prevent duplicate builds when tags are pushed.
+
+  # Or it can be triggered manually.
+  workflow_dispatch:
+
+concurrency:
+  group: release
+  cancel-in-progress: false
+
+jobs:
+  release_metadata:
+    name: Prepare release metadata
+    runs-on: ubuntu-latest
+    outputs:
+      version_number: ${{ steps.release_metadata.outputs.version_number }}
+      tag_name: ${{ steps.release_metadata.outputs.tag_name }}
+      changelog: ${{ steps.release_metadata.outputs.changelog }}
+    steps:
+      - uses: apify/workflows/git-cliff-release@main
+        id: release_metadata
+        name: Prepare release metadata
+        with:
+          release_type: prerelease
+          existing_changelog_path: CHANGELOG.md
+
+  # If github.ref points to a [ci skip] commit, this workflow won't run.
+  # Otherwise, these checks will be launched from the `run_code_checks` workflow.
+  wait_for_checks:
+    name: Wait for code checks to pass
+    runs-on: ubuntu-latest
+    steps:
+      - uses: lewagon/wait-on-check-action@v1.4.1
+        with:
+          ref: ${{ github.ref }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          check-regexp: "( check| tests)$"
+          wait-interval: 5
+
+  update_changelog:
+    name: Update changelog
+    needs: [release_metadata, wait_for_checks]
+    uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main
+    with:
+      version_number: ${{ needs.release_metadata.outputs.version_number }}
+      changelog: ${{ needs.release_metadata.outputs.changelog }}
+    secrets:
+      APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+
+  build_and_deploy_docs:
+    needs: [update_changelog]
+    uses: ./.github/workflows/build_and_deploy_docs.yaml
+    with:
+      ref: ${{ needs.update_changelog.outputs.changelog_commitish }}
+    secrets: inherit
+
+  publish_to_pypi:
+    if: "!startsWith(github.event.head_commit.message, 'ci') && !startsWith(github.event.head_commit.message, 'docs')"
+    name: Publish to PyPI
+    needs: [release_metadata, update_changelog]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write # Required for OIDC authentication.
+    environment:
+      name: pypi
+      url: https://pypi.org/project/crawlee
+    steps:
+      - name: Prepare distribution
+        uses: apify/workflows/prepare-pypi-distribution@main
+        with:
+          package_name: crawlee
+          is_prerelease: "yes"
+          version_number: ${{ needs.release_metadata.outputs.version_number }}
+          ref: ${{ needs.update_changelog.outputs.changelog_commitish }}
+      # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication.
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+      # TODO: add job for publish package to Conda
+      # https://github.com/apify/crawlee-python/issues/104

crawlee-1.0.5b21/.github/workflows/release.yaml

@@ -0,0 +1,121 @@
+name: Create a release
+
+on:
+  # Trigger a stable version release via GitHub's UI, with the ability to specify the type of release.
+  workflow_dispatch:
+    inputs:
+      release_type:
+        description: Release type
+        required: true
+        type: choice
+        default: auto
+        options:
+          - auto
+          - custom
+          - patch
+          - minor
+          - major
+      custom_version:
+        description: The custom version to bump to (only for "custom" type)
+        required: false
+        type: string
+        default: ""
+
+concurrency:
+  group: release
+  cancel-in-progress: false
+
+jobs:
+  release_metadata:
+    name: Prepare release metadata
+    runs-on: ubuntu-latest
+    outputs:
+      version_number: ${{ steps.release_metadata.outputs.version_number }}
+      tag_name: ${{ steps.release_metadata.outputs.tag_name }}
+      changelog: ${{ steps.release_metadata.outputs.changelog }}
+      release_notes: ${{ steps.release_metadata.outputs.release_notes }}
+    steps:
+      - uses: apify/workflows/git-cliff-release@main
+        name: Prepare release metadata
+        id: release_metadata
+        with:
+          release_type: ${{ inputs.release_type }}
+          custom_version: ${{ inputs.custom_version }}
+          existing_changelog_path: CHANGELOG.md
+
+  lint_check:
+    name: Lint check
+    uses: apify/workflows/.github/workflows/python_lint_check.yaml@main
+    with:
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
+
+  type_check:
+    name: Type check
+    uses: apify/workflows/.github/workflows/python_type_check.yaml@main
+    with:
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
+
+  unit_tests:
+    name: Unit tests
+    uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main
+    secrets:
+      httpbin_url: ${{ secrets.APIFY_HTTPBIN_TOKEN && format('https://httpbin.apify.actor?token={0}', secrets.APIFY_HTTPBIN_TOKEN) || 'https://httpbin.org'}}
+    with:
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
+
+  update_changelog:
+    name: Update changelog
+    needs: [release_metadata, lint_check, type_check, unit_tests]
+    uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main
+    with:
+      version_number: ${{ needs.release_metadata.outputs.version_number }}
+      changelog: ${{ needs.release_metadata.outputs.changelog }}
+    secrets:
+      APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+
+  build_and_deploy_docs:
+    needs: [update_changelog]
+    uses: ./.github/workflows/build_and_deploy_docs.yaml
+    with:
+      ref: ${{ needs.update_changelog.outputs.changelog_commitish }}
+    secrets: inherit
+
+  create_github_release:
+    name: Create github release
+    needs: [release_metadata, update_changelog]
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Create release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ needs.release_metadata.outputs.tag_name }}
+          name: ${{ needs.release_metadata.outputs.version_number }}
+          target_commitish: ${{ needs.update_changelog.outputs.changelog_commitish }}
+          body: ${{ needs.release_metadata.outputs.release_notes }}
+
+  publish_to_pypi:
+    name: Publish to PyPI
+    needs: [release_metadata, update_changelog]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write # Required for OIDC authentication.
+    environment:
+      name: pypi
+      url: https://pypi.org/project/crawlee
+    steps:
+      - name: Prepare distribution
+        uses: apify/workflows/prepare-pypi-distribution@main
+        with:
+          package_name: crawlee
+          is_prerelease: ""
+          version_number: ${{ needs.release_metadata.outputs.version_number }}
+          ref: ${{ needs.update_changelog.outputs.changelog_commitish }}
+      # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication.
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+      # TODO: add job for publish package to Conda
+      # https://github.com/apify/crawlee-python/issues/104

crawlee-1.0.5b21/.github/workflows/run_code_checks.yaml

@@ -0,0 +1,43 @@
+name: Run code checks
+
+on:
+  # Trigger code checks on opening a new pull request.
+  # Secrets are only made available to the integration tests job, with a manual approval
+  # step required for PRs from forks. This prevents their potential exposure.
+  pull_request:
+
+  # Pushing to the master branch triggers code checks
+  push:
+    branches:
+      - master
+    tags-ignore:
+      - "**" # Ignore all tags to prevent duplicate checks when tags are pushed.
+
+  # It should also be possible to trigger checks manually
+  workflow_dispatch:
+
+jobs:
+  lint_check:
+    name: Lint check
+    uses: apify/workflows/.github/workflows/python_lint_check.yaml@main
+    with:
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
+
+  type_check:
+    name: Type check
+    uses: apify/workflows/.github/workflows/python_type_check.yaml@main
+    with:
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
+
+  unit_tests:
+    name: Unit tests
+    uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main
+    secrets:
+      httpbin_url: ${{ secrets.APIFY_HTTPBIN_TOKEN && format('https://httpbin.apify.actor?token={0}', secrets.APIFY_HTTPBIN_TOKEN) || 'https://httpbin.org'}}
+    with:
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
+
+  docs_check:
+    name: Docs check
+    uses: apify/workflows/.github/workflows/python_docs_check.yaml@main
+    secrets: inherit

crawlee-1.0.5b21/.github/workflows/templates_e2e_tests.yaml

@@ -0,0 +1,58 @@
+name: Templates end-to-end tests
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 6 * * *'
+
+env:
+  NODE_VERSION: 22
+  PYTHON_VERSION: 3.14
+
+jobs:
+  end_to_end_tests:
+    name: End-to-end tests
+    strategy:
+      fail-fast: false
+      max-parallel: 12
+      matrix:
+        crawler-type: ["playwright_camoufox", "playwright", "parsel", "beautifulsoup"]
+        http-client: [ "httpx", "curl_impersonate"]
+        package-manager: ["pip", "uv", "poetry"]
+
+    runs-on: "ubuntu-latest"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Setup node
+        uses: actions/setup-node@v6
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Install dependencies
+        run: npm install -g apify-cli
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      # installed to be able to patch crawlee in the poetry.lock with custom wheel file for poetry based templates
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up uv package manager
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      # Sync the project, but no need to install the browsers into the test runner environment.
+      - name: Install Python dependencies
+        run: make install-sync
+
+      - name: Run templates end-to-end tests
+        run: make e2e-templates-tests args="-m ${{ matrix.http-client }} and ${{ matrix.crawler-type }} and ${{ matrix.package-manager }}"
+        env:
+          APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_API_TOKEN }}

crawlee-1.0.5b21/.github/workflows/update_new_issue.yaml

@@ -0,0 +1,25 @@
+name: Update new issue
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  label_issues:
+    name: Label issues
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+
+    steps:
+      # Add the "t-tooling" label to all new issues
+      - uses: actions/github-script@v8
+        with:
+          script: |
+            github.rest.issues.addLabels({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ["t-tooling"]
+            })

crawlee-1.0.5b21/.gitignore

@@ -0,0 +1,55 @@
+# Cache
+__pycache__
+.mypy_cache
+.pytest_cache
+.ruff_cache
+
+# Virtual envs
+.venv
+.direnv
+.envrc
+.python-version
+
+# Other Python tools
+.ropeproject
+
+# Mise
+mise.toml
+.mise.toml
+
+# Egg and build artifacts
+*.egg-info/
+*.egg
+dist/
+build/
+
+# Coverage reports
+.coverage*
+htmlcov
+
+# IDE, editors
+.vscode
+.idea
+*~
+.DS_Store
+.nvim.lua
+Session.vim
+
+# Docs
+docs/changelog.md
+
+# Website build artifacts, node dependencies
+website/build
+website/node_modules
+website/.yarn
+website/.docusaurus
+website/api-typedoc-generated.json
+website/apify-shared-docspec-dump.jsonl
+website/docspec-dump.jsonl
+website/module_shortcuts.json
+website/typedoc-types*
+# npm lockfile (we use yarn)
+website/package-lock.json
+
+# Default directory for memory storage
+storage/