testdriverai 7.3.2 → 7.3.4

package/.claude/settings.local.json

@@ -0,0 +1,7 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(curl:*)"
+    ]
+  }
+}

package/.env.example

@@ -0,0 +1,4 @@
+TD_API_KEY=
+TD_API_ROOT=
+
+TWOCAPTCHA_API_KEY=

package/.github/workflows/acceptance-linux-scheduled.yaml

@@ -0,0 +1,45 @@
+name: Scheduled Acceptance Tests (Linux)
+
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  schedule:
+    # Every 6 hours
+    - cron: "0 */6 * * *"
+  workflow_dispatch: # Allow manual trigger
+
+jobs:
+  test-linux:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Install Sentry CLI
+        run: npm install -g @sentry/cli
+
+      - name: Run Linux tests with Sentry Cron monitoring
+        run: |
+          sentry-cli monitors run testdriver-linux-acceptance -- vitest run examples/*.test.mjs
+        env:
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+          TD_API_KEY: ${{ secrets.TD_API_KEY }}
+          TD_OS: linux
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+
+      - name: Upload test results to Sentry Prevent
+        if: ${{ !cancelled() }}
+        uses: getsentry/prevent-action@v0
+        with:
+          token: ${{ secrets.SENTRY_PREVENT_TOKEN }}

package/.github/workflows/acceptance-windows-scheduled.yaml

@@ -0,0 +1,54 @@
+name: Scheduled Acceptance Tests (Windows)
+
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  schedule:
+    # Every day at midnight UTC
+    - cron: "0 0 * * *"
+  workflow_dispatch: # Allow manual trigger
+
+jobs:
+  test-windows:
+    uses: ./.github/workflows/windows-self-hosted.yaml
+    secrets:
+      TD_API_KEY: ${{ secrets.TD_API_KEY }}
+      TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+      TD_WEBSITE: ${{ secrets.TD_WEBSITE }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+  sentry-and-publish:
+    needs: test-windows
+    runs-on: ubuntu-latest
+    if: always()
+
+    steps:
+      - name: Install Sentry CLI
+        run: npm install -g @sentry/cli
+
+      - name: Report to Sentry Cron
+        run: |
+          sentry-cli monitors run testdriver-windows-acceptance -- echo "Windows tests completed"
+        env:
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+
+      - name: Download test results
+        uses: actions/download-artifact@v4
+        with:
+          name: test-results-windows
+
+      - name: Upload test results to Sentry Prevent
+        if: ${{ !cancelled() }}
+        uses: getsentry/prevent-action@v0
+        with:
+          token: ${{ secrets.SENTRY_PREVENT_TOKEN }}
+
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          files: test-report.junit.xml
+          check_name: Test Results (Windows Scheduled)

package/.github/workflows/acceptance.yaml

@@ -0,0 +1,106 @@
+name: Acceptance Tests
+permissions:
+  id-token: write
+  contents: write
+  pull-requests: write
+  checks: write
+on:
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-linux:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run Linux tests
+        run: npx vitest run examples/*.test.mjs 2>&1 | tee test-output.log
+        env:
+          TD_API_KEY: ${{ secrets.TD_API_KEY }}
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          TD_OS: linux
+
+      - name: Extract example URLs
+        if: success()
+        run: node docs/_scripts/extract-example-urls.js --file=test-output.log
+
+      - name: Generate example docs
+        if: success()
+        run: node docs/_scripts/generate-examples.js --skip-ai
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+      - name: Commit updated docs
+        if: success()
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add docs/
+          git diff --staged --quiet || git commit -m "docs: update example documentation [skip ci]"
+          git push || echo "No changes to push"
+
+      - name: Upload test results to Sentry Prevent
+        if: ${{ !cancelled() }}
+        uses: getsentry/prevent-action@v0
+        with:
+          token: ${{ secrets.SENTRY_PREVENT_TOKEN }}
+
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          files: test-report.junit.xml
+          comment_mode: always
+          check_name: Test Results (Linux)
+
+  test-windows:
+    if: contains(github.event.pull_request.labels.*.name, 'test-windows')
+    uses: ./.github/workflows/windows-self-hosted.yaml
+    with:
+      test_pattern: "examples/*.test.mjs"
+    secrets:
+      TD_API_KEY: ${{ secrets.TD_API_KEY }}
+      TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+      TD_WEBSITE: ${{ secrets.TD_WEBSITE }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+  publish-windows-results:
+    needs: test-windows
+    runs-on: ubuntu-latest
+    if: always() && contains(github.event.pull_request.labels.*.name, 'test-windows')
+
+    steps:
+      - name: Download test results
+        uses: actions/download-artifact@v4
+        with:
+          name: test-results-windows
+
+      - name: Upload test results to Sentry Prevent
+        if: ${{ !cancelled() }}
+        uses: getsentry/prevent-action@v0
+        with:
+          token: ${{ secrets.SENTRY_PREVENT_TOKEN }}
+
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          files: test-report.junit.xml
+          comment_mode: always
+          check_name: Test Results (Windows)

package/.github/workflows/publish.yaml

@@ -0,0 +1,75 @@
+name: Publish
+permissions:
+  contents: write
+  id-token: write # Required for OIDC
+on:
+  push:
+    branches: [main]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          registry-url: "https://registry.npmjs.org/"
+
+      - name: Configure Git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build MCP Server
+        run: npm run build:mcp
+
+      - name: Bump version (patch)
+        run: npm version patch --no-git-tag-version
+
+      - name: Generate Changelog
+        run: |
+          npx conventional-changelog-cli -p angular -i CHANGELOG.md -s
+          git add CHANGELOG.md
+
+      - name: Commit and push version bump
+        run: |
+          git add package.json package-lock.json
+          VERSION=$(node -p "require('./package.json').version")
+          git commit -m "chore(release): ${VERSION}"
+          git tag "v${VERSION}"
+          
+          # Pull any changes that happened since checkout
+          git pull --rebase origin main
+          
+          git push origin main
+          git push origin "v${VERSION}"
+          echo "VERSION=${VERSION}" >> $GITHUB_ENV
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v1
+        with:
+          tag_name: v${{ env.VERSION }}
+          generate_release_notes: true
+          prerelease: false
+
+      - name: Debug NPM Token
+        run: |
+          echo "NPM_TOKEN is set: ${{ secrets.NPM_TOKEN != '' }}"
+          echo "NPM_TOKEN first 4 chars: ${NPM_TOKEN:0:4}..."
+        env:
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      - name: Publish to npm
+        run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

package/.github/workflows/test-init.yml

@@ -0,0 +1,157 @@
+name: Test Init Command
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+  workflow_dispatch:
+
+jobs:
+  test-init:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - name: Checkout CLI repository
+      uses: actions/checkout@v4
+    
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+        cache: 'npm'
+    
+    - name: Install CLI dependencies
+      run: npm ci
+    
+    - name: Pack local testdriverai
+      run: |
+        npm pack
+        mv testdriverai-*.tgz /tmp/testdriverai-local.tgz
+    
+    - name: Create test directory
+      run: |
+        mkdir -p /tmp/test-init-project
+        cd /tmp/test-init-project
+    
+    - name: Run init command (skip prompts)
+      working-directory: /tmp/test-init-project
+      run: |
+        # Create .env with API key first to skip the prompt
+        echo "TD_API_KEY=${{ secrets.TD_API_KEY }}" > .env
+        
+        # Run init command using the CLI from the repo
+        node ${{ github.workspace }}/bin/testdriverai.js init
+      env:
+        TD_API_KEY: ${{ secrets.TD_API_KEY }}
+    
+    - name: Replace npm testdriverai with local build
+      working-directory: /tmp/test-init-project
+      run: |
+        # Remove npm-installed version and install local tarball
+        npm uninstall testdriverai
+        npm install /tmp/testdriverai-local.tgz
+    
+    - name: Verify project structure
+      working-directory: /tmp/test-init-project
+      run: |
+        echo "Checking generated files..."
+        
+        # Check for package.json
+        if [ ! -f "package.json" ]; then
+          echo "❌ package.json not found"
+          exit 1
+        fi
+        echo "✓ package.json exists"
+        
+        # Check for vitest config
+        if [ ! -f "vitest.config.js" ]; then
+          echo "❌ vitest.config.js not found"
+          exit 1
+        fi
+        echo "✓ vitest.config.js exists"
+        
+        # Check for test file
+        if [ ! -f "tests/example.test.js" ]; then
+          echo "❌ tests/example.test.js not found"
+          exit 1
+        fi
+        echo "✓ tests/example.test.js exists"
+        
+        # Check for .env file
+        if [ ! -f ".env" ]; then
+          echo "❌ .env not found"
+          exit 1
+        fi
+        echo "✓ .env exists"
+        
+        # Check for .gitignore
+        if [ ! -f ".gitignore" ]; then
+          echo "❌ .gitignore not found"
+          exit 1
+        fi
+        echo "✓ .gitignore exists"
+        
+        # Check for GitHub workflow
+        if [ ! -f ".github/workflows/testdriver.yml" ]; then
+          echo "❌ .github/workflows/testdriver.yml not found"
+          exit 1
+        fi
+        echo "✓ .github/workflows/testdriver.yml exists"
+    
+    - name: Verify vitest config contents
+      working-directory: /tmp/test-init-project
+      run: |
+        echo "Checking vitest.config.js contents..."
+        
+        # Check for TestDriver reporter
+        if ! grep -q "TestDriver()" vitest.config.js; then
+          echo "❌ TestDriver reporter not found in vitest.config.js"
+          cat vitest.config.js
+          exit 1
+        fi
+        echo "✓ TestDriver reporter is configured"
+        
+        # Check for setupFiles
+        if ! grep -q "setupFiles.*testdriverai/vitest/setup" vitest.config.js; then
+          echo "❌ setupFiles not configured correctly"
+          cat vitest.config.js
+          exit 1
+        fi
+        echo "✓ setupFiles is configured"
+    
+    - name: Verify test file contents
+      working-directory: /tmp/test-init-project
+      run: |
+        echo "Checking test file contents..."
+        
+        # Check for .provision usage
+        if ! grep -q "\.provision\.chrome" tests/example.test.js; then
+          echo "❌ Test does not use .provision.chrome"
+          cat tests/example.test.js
+          exit 1
+        fi
+        echo "✓ Test uses .provision.chrome"
+        
+        # Check for TestDriver import
+        if ! grep -q "from 'testdriverai/vitest/hooks'" tests/example.test.js; then
+          echo "❌ Test does not import from testdriverai/vitest/hooks"
+          cat tests/example.test.js
+          exit 1
+        fi
+        echo "✓ Test imports TestDriver from vitest/hooks"
+    
+    - name: Run the generated test
+      working-directory: /tmp/test-init-project
+      run: npm test
+      env:
+        TD_API_KEY: ${{ secrets.TD_API_KEY }}
+    
+    - name: Upload test results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-init-results
+        path: /tmp/test-init-project/test-results/
+        retention-days: 7
+        if-no-files-found: warn

package/.github/workflows/testdriver.yml

@@ -0,0 +1,170 @@
+name: TestDriver.ai Tests
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      pull-requests: write # Required to post comments on PRs
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run TestDriver.ai tests
+        env:
+          TD_API_KEY: ${{ secrets.TD_API_KEY }}
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: vitest run
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results
+          path: test-results/
+          retention-days: 30
+
+      # Init command test - only runs on PRs, not on main/master
+      - name: Create test directory for init
+        if: github.event_name == 'pull_request'
+        run: |
+          mkdir -p /tmp/test-init-project
+          cd /tmp/test-init-project
+
+      - name: Run init command (skip prompts)
+        if: github.event_name == 'pull_request'
+        working-directory: /tmp/test-init-project
+        run: |
+          # Create .env with API key first to skip the prompt
+          echo "TD_API_KEY=${{ secrets.TD_API_KEY }}" > .env
+
+          # Run init command using the CLI from the repo
+          node ${{ github.workspace }}/bin/testdriverai.js init
+        env:
+          TD_API_KEY: ${{ secrets.TD_API_KEY }}
+
+      - name: Verify project structure
+        if: github.event_name == 'pull_request'
+        working-directory: /tmp/test-init-project
+        run: |
+          echo "Checking generated files..."
+
+          # Check for package.json
+          if [ ! -f "package.json" ]; then
+            echo "❌ package.json not found"
+            exit 1
+          fi
+          echo "✓ package.json exists"
+
+          # Check for vitest config
+          if [ ! -f "vitest.config.js" ]; then
+            echo "❌ vitest.config.js not found"
+            exit 1
+          fi
+          echo "✓ vitest.config.js exists"
+
+          # Check for test file
+          if [ ! -f "tests/example.test.js" ]; then
+            echo "❌ tests/example.test.js not found"
+            exit 1
+          fi
+          echo "✓ tests/example.test.js exists"
+
+          # Check for .env file
+          if [ ! -f ".env" ]; then
+            echo "❌ .env not found"
+            exit 1
+          fi
+          echo "✓ .env exists"
+
+          # Check for .gitignore
+          if [ ! -f ".gitignore" ]; then
+            echo "❌ .gitignore not found"
+            exit 1
+          fi
+          echo "✓ .gitignore exists"
+
+          # Check for GitHub workflow
+          if [ ! -f ".github/workflows/testdriver.yml" ]; then
+            echo "❌ .github/workflows/testdriver.yml not found"
+            exit 1
+          fi
+          echo "✓ .github/workflows/testdriver.yml exists"
+
+      - name: Verify vitest config contents
+        if: github.event_name == 'pull_request'
+        working-directory: /tmp/test-init-project
+        run: |
+          echo "Checking vitest.config.js contents..."
+
+          # Check for TestDriver reporter
+          if ! grep -q "TestDriver()" vitest.config.js; then
+            echo "❌ TestDriver reporter not found in vitest.config.js"
+            cat vitest.config.js
+            exit 1
+          fi
+          echo "✓ TestDriver reporter is configured"
+
+          # Check for setupFiles
+          if ! grep -q "setupFiles.*testdriverai/vitest/setup" vitest.config.js; then
+            echo "❌ setupFiles not configured correctly"
+            cat vitest.config.js
+            exit 1
+          fi
+          echo "✓ setupFiles is configured"
+
+      - name: Verify test file contents
+        if: github.event_name == 'pull_request'
+        working-directory: /tmp/test-init-project
+        run: |
+          echo "Checking test file contents..."
+
+          # Check for .provision usage
+          if ! grep -q "\.provision\.chrome" tests/example.test.js; then
+            echo "❌ Test does not use .provision.chrome"
+            cat tests/example.test.js
+            exit 1
+          fi
+          echo "✓ Test uses .provision.chrome"
+
+          # Check for TestDriver import
+          if ! grep -q "from 'testdriverai/vitest/hooks'" tests/example.test.js; then
+            echo "❌ Test does not import from testdriverai/vitest/hooks"
+            cat tests/example.test.js
+            exit 1
+          fi
+          echo "✓ Test imports TestDriver from vitest/hooks"
+
+      - name: Run the generated test
+        if: github.event_name == 'pull_request'
+        working-directory: /tmp/test-init-project
+        run: npm test
+        env:
+          TD_API_KEY: ${{ secrets.TD_API_KEY }}
+
+      - name: Upload init test results
+        if: always() && github.event_name == 'pull_request'
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-init-results
+          path: /tmp/test-init-project/test-results/
+          retention-days: 7
+          if-no-files-found: warn

package/.github/workflows/windows-self-hosted.yaml

@@ -0,0 +1,82 @@
+name: Windows Self-Hosted Tests
+
+on:
+  workflow_call:
+    inputs:
+      test_pattern:
+        description: "Test file pattern to run"
+        required: false
+        type: string
+        default: "examples/*.test.mjs"
+    secrets:
+      TD_API_KEY:
+        required: true
+      TWOCAPTCHA_API_KEY:
+        required: true
+      TD_WEBSITE:
+        required: false
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Debug Environment
+        run: |
+          echo "Checking environment variables..."
+          if [ -n "${{ secrets.TWOCAPTCHA_API_KEY }}" ]; then
+            echo "TWOCAPTCHA_API_KEY is set (length: ${#TWOCAPTCHA_API_KEY})"
+          else
+            echo "TWOCAPTCHA_API_KEY is NOT set"
+          fi
+        env:
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+
+      - name: Run Windows tests with self-hosted instances
+        run: vitest run ${{ inputs.test_pattern }}
+        env:
+          TD_API_KEY: ${{ secrets.TD_API_KEY }}
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          TD_WEBSITE: ${{ secrets.TD_WEBSITE }}
+          TD_OS: windows
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: us-east-2
+          AWS_LAUNCH_TEMPLATE_ID: lt-0ef9bf26a945fb442
+          AMI_ID: ami-0dd5fa241273a7d50
+          RESOLUTION: 1920x1080
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-windows
+          path: test-report.junit.xml
+          retention-days: 30
+
+      - name: Upload TestDriver AI CLI logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: testdriverai-cli-logs-windows
+          path: /tmp/testdriverai-cli-*.log
+          if-no-files-found: warn
+          retention-days: 30