Test YAML

version: "1.0"

# File-level defaults applied to every case.
# Per-case fields override these.
defaults:
  runs: 5                     # invoke the agent 5 times per case
  success_threshold: 80       # 4/5 must pass for the case to pass
  timeout_s: 60               # per-invocation wall clock

tests:
  - name: adds_two_numbers
    payload: '{"message": "add 4 and 6", "a": 4, "b": 6}'
    expected_result: status == "completed"
    expected_tool_calls:
      - math.calculator.add: invocations >= 1

  - name: plain_message_no_tools
    payload: "say hello"
    expected_result: status == "completed"
    expected_no_tool_calls:
      - math.calculator.add

  - name: high_concurrency_smoke
    payload: '{"message": "stress ping"}'
    runs: 20                  # override file default for this case
    success_threshold: 95
    expected_result: status == "completed"

# A second suite for the stress-tester agent. The filename can't also be
# stress-tester.yaml (it's already taken), so we point at the agent
# explicitly and give the file a more descriptive name.
agent: stress-tester

defaults:
  runs: 50
  success_threshold: 90

tests:
  - name: sustained_burst
    payload: '{"message": "stress ping"}'
    expected_result: status == "completed"