testdriverai 6.2.2 → 7.1.0

package/.github/workflows/acceptance-linux.yml

@@ -0,0 +1,75 @@
+name: v6 Linux
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - "docs/**"
+  pull_request:
+    branches:
+      - main
+
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  # Job to gather all test files
+  test-setup:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 22
+          cache: npm
+      - run: npm ci
+
+  gather:
+    name: Gather Test Files
+    runs-on: ubuntu-latest
+    outputs:
+      test_files: ${{ steps.test_list.outputs.files }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      - name: Find all test files
+        id: test_list
+        run: |
+          FILES=$(ls ./testdriver/acceptance/*.yaml)
+          FILENAMES=$(basename -a $FILES)
+          FILES_JSON=$(echo "$FILENAMES" | jq -R -s -c 'split("\n")[:-1]')
+          echo "files=$FILES_JSON" >> $GITHUB_OUTPUT
+  test:
+    needs:
+      - gather
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test: ${{ fromJson(needs.gather.outputs.test_files) }}
+      max-parallel: 8
+      fail-fast: false
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: NODE_ENV=production npm ci
+      - name: Run test in headless mode
+        run: node bin/testdriverai.js run testdriver/acceptance/${{ matrix.test }} --junit=out.xml
+        env:
+          FORCE_COLOR: 3
+          TD_API_KEY: 49c2996c-0687-40bf-9f2c-5d2e55b2a2c6
+          TD_WEBSITE: https://testdriver-sandbox.vercel.app
+          TD_THIS_FILE: ${{ matrix.test }}
+          TD_API_ROOT: "https://replayable-dev-ian-mac-m1-16.ngrok.io"

package/.github/workflows/acceptance-sdk-tests.yml

@@ -0,0 +1,133 @@
+name: Acceptance SDK Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches: [main]
+    paths-ignore:
+      - "docs/**"
+  # So that we can manually trigger tests when there's flake
+  workflow_dispatch:
+
+jobs:
+  test-workflow:
+    name: Run SDK Tests - ${{ matrix.platform }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [linux, windows]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run ESLint
+        run: npx eslint . --max-warnings 0
+
+      - name: Run Prettier
+        run: npx prettier --check .
+
+      - name: Run SDK tests with Vitest - ${{ matrix.platform }}
+        run: npx vitest run testdriver/acceptance-sdk/*.test.mjs
+        env:
+          FORCE_COLOR: 3
+          TD_API_KEY: ${{ secrets.TESTDRIVER_API_KEY }}
+          TEST_PLATFORM: ${{ matrix.platform }}
+          VERBOSE: true
+          LOGGING: true
+        continue-on-error: true
+
+      - name: Generate GitHub Summary
+        if: always()
+        run: |
+          echo "# 🧪 TestDriver SDK Test Results" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          if [ -f test-results/results.json ]; then
+            node -e "
+              const fs = require('fs');
+              const results = JSON.parse(fs.readFileSync('test-results/results.json', 'utf8'));
+              
+              const total = results.numTotalTests || 0;
+              const passed = results.numPassedTests || 0;
+              const failed = results.numFailedTests || 0;
+              const skipped = results.numPendingTests || 0;
+              const duration = ((results.testResults?.[0]?.endTime - results.testResults?.[0]?.startTime) / 1000 || 0).toFixed(2);
+              
+              console.log('## 📊 Overview\n');
+              console.log('| Metric | Count |');
+              console.log('|--------|-------|');
+              console.log('| ✅ Passed | ' + passed + ' |');
+              console.log('| ❌ Failed | ' + failed + ' |');
+              console.log('| ⏭️ Skipped | ' + skipped + ' |');
+              console.log('| 📝 Total | ' + total + ' |');
+              console.log('| ⏱️ Duration | ' + duration + 's |');
+              console.log('');
+              
+              if (failed > 0) {
+                console.log('## ❌ Failed Tests\n');
+                results.testResults?.forEach(file => {
+                  file.assertionResults?.filter(test => test.status === 'failed').forEach(test => {
+                    console.log('### ' + test.fullName);
+                    console.log('**File:** \`' + file.name + '\`');
+                    console.log('');
+                    if (test.failureMessages?.length > 0) {
+                      console.log('**Error:**');
+                      console.log('\`\`\`');
+                      console.log(test.failureMessages.join('\n').substring(0, 1000));
+                      console.log('\`\`\`');
+                      console.log('');
+                    }
+                  });
+                });
+              }
+              
+              if (passed > 0) {
+                console.log('## ✅ Passed Tests\n');
+                results.testResults?.forEach(file => {
+                  const passedTests = file.assertionResults?.filter(test => test.status === 'passed') || [];
+                  if (passedTests.length > 0) {
+                    console.log('### ' + file.name.split('/').pop());
+                    passedTests.forEach(test => {
+                      console.log('- ✅ ' + test.title);
+                    });
+                    console.log('');
+                  }
+                });
+              }
+            " >> $GITHUB_STEP_SUMMARY
+          else
+            echo "⚠️ No test results found" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: Test Summary (JUnit)
+        uses: test-summary/action@v2
+        with:
+          paths: "test-results/junit.xml"
+        if: always()
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-${{ matrix.platform }}
+          path: |
+            test-results/junit.xml
+            test-results/results.json
+            test-results/index.html
+          retention-days: 7
+          if-no-files-found: ignore

package/.vscode/settings.json

@@ -6,5 +6,9 @@
   "editor.defaultFormatter": "esbenp.prettier-vscode",
   "yaml.schemas": {
     "https://raw.githubusercontent.com/testdriverai/testdriverai/main/schema.json": "file:///Users/kid/Desktop/td/internal/testdriverai/testdriver.yaml"
-  }
+  },
+  "vitest.enable": true,
+  "vitest.commandLine": "npx vitest --watch",
+  "vitest.include": ["**/testdriver/acceptance-sdk/*.test.mjs"],
+  "vitest.exclude": []
 }