👷 Improve Playwright CI speed: sharding (paralel runs), run in Docker to use cache, use env vars (#1405)

.github/workflows/playwright.yml

@@ -16,10 +16,36 @@ on:
         default: 'false'
 
 jobs:
+  changes:
+    runs-on: ubuntu-latest
+    # Set job outputs to values from filter step
+    outputs:
+      changed: ${{ steps.filter.outputs.changed }}
+    steps:
+    - uses: actions/checkout@v4
+    # For pull requests it's not necessary to checkout the code but for the main branch it is
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        filters: |
+          changed:
+            - backend/**
+            - frontend/**
+            - .env
+            - docker-compose*.yml
+            - .github/workflows/playwright.yml
 
-  test:
+  test-playwright:
+    needs:
+      - changes
+    if: ${{ needs.changes.outputs.changed == 'true' }}
     timeout-minutes: 60
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
+      fail-fast: false
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-node@v4
@@ -33,35 +59,61 @@ jobs:
       if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }}
       with:
         limit-access-to-actor: true
+    - run: docker compose build
+    - run: docker compose down -v --remove-orphans
+    - name: Run Playwright tests
+      run: docker compose run --rm playwright npx playwright test --fail-on-flaky-tests --trace=retain-on-failure --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+    - run: docker compose down -v --remove-orphans
+    - name: Upload blob report to GitHub Actions Artifacts
+      if: ${{ !cancelled() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: blob-report-${{ matrix.shardIndex }}
+        path: frontend/blob-report
+        include-hidden-files: true
+        retention-days: 1
+
+  merge-playwright-reports:
+    needs:
+      - test-playwright
+      - changes
+    # Merge reports after playwright-tests, even if some shards have failed
+    if: ${{ !cancelled() && needs.changes.outputs.changed == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 20
     - name: Install dependencies
       run: npm ci
       working-directory: frontend
-    - name: Install Playwright Browsers
+    - name: Download blob reports from GitHub Actions Artifacts
-      run: npx playwright install --with-deps
+      uses: actions/download-artifact@v4
-      working-directory: frontend
-    - run: docker compose build
-    - run: docker compose down -v --remove-orphans
-    - run: docker compose up -d --wait backend mailcatcher
-    - name: Run Playwright tests
-      run: npx playwright test --fail-on-flaky-tests --trace=retain-on-failure
-      working-directory: frontend
-    - run: docker compose down -v --remove-orphans
-    - uses: actions/upload-artifact@v4
-      if: always()
       with:
-        name: playwright-report
+        path: frontend/all-blob-reports
-        path: frontend/playwright-report/
+        pattern: blob-report-*
+        merge-multiple: true
+    - name: Merge into HTML Report
+      run: npx playwright merge-reports --reporter html ./all-blob-reports
+      working-directory: frontend
+    - name: Upload HTML report
+      uses: actions/upload-artifact@v4
+      with:
+        name: html-report--attempt-${{ github.run_attempt }}
+        path: frontend/playwright-report
         retention-days: 30
         include-hidden-files: true
 
   # https://github.com/marketplace/actions/alls-green#why
-  e2e-alls-green:  # This job does nothing and is only used for the branch protection
+  alls-green-playwright:  # This job does nothing and is only used for the branch protection
     if: always()
     needs:
-      - test
+      - test-playwright
     runs-on: ubuntu-latest
     steps:
       - name: Decide whether the needed jobs succeeded or failed
         uses: re-actors/alls-green@release/v1
         with:
           jobs: ${{ toJSON(needs) }}
+          allowed-skips: test-playwright

.github/workflows/test-docker-compose.yml

@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@v4
       - run: docker compose build
       - run: docker compose down -v --remove-orphans
-      - run: docker compose up -d --wait
+      - run: docker compose up -d --wait backend frontend adminer
       - name: Test backend is up
         run: curl http://localhost:8000/api/v1/utils/health-check
       - name: Test frontend is up

docker-compose.override.yml

@@ -102,6 +102,31 @@ services:
         - VITE_API_URL=http://localhost:8000
         - NODE_ENV=development
 
+  playwright:
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile.playwright
+      args:
+        - VITE_API_URL=http://backend:8000
+        - NODE_ENV=production
+    ipc: host
+    depends_on:
+      - backend
+      - mailcatcher
+    env_file:
+      - .env
+    environment:
+      - VITE_API_URL=http://backend:8000
+      - MAILCATCHER_HOST=http://mailcatcher:1080
+      # For the reports when run locally
+      - PLAYWRIGHT_HTML_HOST=0.0.0.0
+      - CI=${CI}
+    volumes:
+      - ./frontend/blob-report:/app/blob-report
+      - ./frontend/test-results:/app/test-results
+    ports:
+      - 9323:9323
+
 networks:
   traefik-public:
     # For local dev, don't expect an external Traefik network

frontend/.env

@@ -1 +1,2 @@
 VITE_API_URL=http://localhost:8000
+MAILCATCHER_HOST=http://localhost:1080

frontend/Dockerfile.playwright

@@ -0,0 +1,13 @@
+FROM node:20
+
+WORKDIR /app
+
+COPY package*.json /app/
+
+RUN npm install
+
+RUN npx -y playwright install --with-deps
+
+COPY ./ /app/
+
+ARG VITE_API_URL=${VITE_API_URL}

frontend/playwright.config.ts

@@ -1,11 +1,10 @@
 import { defineConfig, devices } from '@playwright/test';
-
+import 'dotenv/config'
 
 /**
  * Read environment variables from file.
  * https://github.com/motdotla/dotenv
  */
-// require('dotenv').config();
 
 /**
  * See https://playwright.dev/docs/test-configuration.
@@ -21,7 +20,7 @@ export default defineConfig({
   /* Opt out of parallel tests on CI. */
   workers: process.env.CI ? 1 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'html',
+  reporter: process.env.CI ? 'blob' : 'html',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Base URL to use in actions like `await page.goto('/')`. */

frontend/src/routes/_layout/admin.tsx

@@ -49,7 +49,7 @@ function UsersTable() {
   const { page } = Route.useSearch()
   const navigate = useNavigate({ from: Route.fullPath })
   const setPage = (page: number) =>
-    navigate({ search: (prev) => ({ ...prev, page }) })
+    navigate({ search: (prev: {[key: string]: string}) => ({ ...prev, page }) })
 
   const {
     data: users,

frontend/src/routes/_layout/items.tsx

@@ -45,7 +45,7 @@ function ItemsTable() {
   const { page } = Route.useSearch()
   const navigate = useNavigate({ from: Route.fullPath })
   const setPage = (page: number) =>
-    navigate({ search: (prev) => ({ ...prev, page }) })
+    navigate({ search: (prev: {[key: string]: string}) => ({ ...prev, page }) })
 
   const {
     data: items,

frontend/tests/reset-password.spec.ts

@@ -50,7 +50,9 @@ test("User can reset password successfully using the link", async ({
     timeout: 5000,
   })
 
-  await page.goto(`http://localhost:1080/messages/${emailData.id}.html`)
+  await page.goto(
+    `${process.env.MAILCATCHER_HOST}/messages/${emailData.id}.html`,
+  )
 
   const selector = 'a[href*="/reset-password?token="]'
 
@@ -103,7 +105,9 @@ test("Weak new password validation", async ({ page, request }) => {
     timeout: 5000,
   })
 
-  await page.goto(`http://localhost:1080/messages/${emailData.id}.html`)
+  await page.goto(
+    `${process.env.MAILCATCHER_HOST}/messages/${emailData.id}.html`,
+  )
 
   const selector = 'a[href*="/reset-password?token="]'
   let url = await page.getAttribute(selector, "href")

frontend/tests/utils/mailcatcher.ts

@@ -10,7 +10,7 @@ async function findEmail({
   request,
   filter,
 }: { request: APIRequestContext; filter?: (email: Email) => boolean }) {
-  const response = await request.get("http://localhost:1080/messages")
+  const response = await request.get(`${process.env.MAILCATCHER_HOST}/messages`)
 
   let emails = await response.json()
 

frontend/tsconfig.json

@@ -20,6 +20,6 @@
     "noUnusedParameters": true,
     "noFallthroughCasesInSwitch": true
   },
-  "include": ["src", "*.ts", "**/*.ts"],
+  "include": ["src/**/*.ts", "tests/**/*.ts", "playwright.config.ts"],
   "references": [{ "path": "./tsconfig.node.json" }]
 }