Using Playwright Agents with Azure DevOps

// Traditional Playwright (scripted)
await page.goto('/checkout')
await page.fill('[name="email"]', 'user@test.com')
await page.click('[data-testid="submit"]')
await expect(page.locator('.confirmation')).toBeVisible()
 
// Agentic Playwright (goal-based — experimental)
await agent.accomplish('Complete a checkout with email user@test.com and verify confirmation')

steps:
  - script: npm ci
  - script: npx playwright install chromium
  - script: |
      # Install Playwright MCP
      npm install @playwright/mcp
    displayName: Install Playwright MCP
 
  - script: |
      node scripts/agent-test.js
    displayName: Run agent tests
    env:
      OPENAI_API_KEY: $(OPENAI_API_KEY)
      BASE_URL: $(STAGING_URL)

// scripts/agent-test.js
import { chromium } from '@playwright/test'
import { PlaywrightMCP } from '@playwright/mcp'
import OpenAI from 'openai'
 
const browser = await chromium.launch({ headless: true })
const page = await browser.newPage()
const mcp = new PlaywrightMCP(page)
const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY })
 
// Navigate to the app
await page.goto(process.env.BASE_URL)
 
// Give the AI agent a test objective
const result = await openai.chat.completions.create({
  model: 'gpt-4o',
  tools: mcp.tools,
  messages: [{
    role: 'user',
    content: `
      Test the checkout flow on ${process.env.BASE_URL}.
      Specifically verify:
      1. Empty cart prevents checkout
      2. Adding an item allows checkout
      3. Valid payment method completes the order
      Report any issues found as JSON.
    `
  }],
  tool_choice: 'auto',
})
 
// Agent executes actions via MCP tools
const report = await mcp.executeToolCalls(result.choices[0].message.tool_calls)
console.log(JSON.stringify(report, null, 2))
 
await browser.close()

// Agent-based exploratory regression
const explorationTargets = [
  'Checkout flow — payment edge cases',
  'User profile — edge cases around name and email updates',
  'Search — boundary conditions on filters',
]
 
const findings = []
 
for (const target of explorationTargets) {
  await page.goto(process.env.BASE_URL)
  
  const result = await openai.chat.completions.create({
    model: 'gpt-4o',
    tools: mcp.tools,
    messages: [{
      role: 'user',
      content: `
        Explore "${target}" on this application.
        Look for: error messages that don't make sense, UI that breaks,
        actions that have no effect, unexpected page state.
        Report findings as a JSON array with: description, severity, steps_to_reproduce.
      `
    }],
  })
 
  const sessionFindings = JSON.parse(/* extract from result */)
  findings.push(...sessionFindings)
}
 
// Write findings to Azure DevOps as work items
for (const finding of findings.filter(f => f.severity !== 'cosmetic')) {
  await createAzureDevOpsBug(finding)
}

// Convert agent findings to JUnit XML
import { Builder } from 'xml2js'
 
function agentFindingsToJUnit(findings: Finding[]): string {
  const failures = findings.filter(f => f.type === 'failure')
  const passes   = findings.filter(f => f.type === 'pass')
 
  return new Builder().buildObject({
    testsuites: {
      testsuite: {
        $: { name: 'Agent Tests', tests: findings.length, failures: failures.length },
        testcase: findings.map(f => ({
          $: { name: f.description, classname: 'Agent' },
          ...(f.type === 'failure' ? {
            failure: { $: { message: f.description }, _: f.steps_to_reproduce }
          } : {})
        }))
      }
    }
  })
}

- task: PublishTestResults@2
  inputs:
    testResultsFormat: JUnit
    testResultsFiles: agent-results/results.xml
    testRunTitle: Agent Testing — $(Build.BuildNumber)
  condition: always()