{
  "cells": {
    "amazon/nova-2-lite-v1/control": {
      "conversations": 50,
      "wrong": 7,
      "answered": 2000
    },
    "amazon/nova-2-lite-v1/break": {
      "conversations": 50,
      "wrong": 1,
      "answered": 2000
    },
    "amazon/nova-2-lite-v1/pause": {
      "conversations": 50,
      "wrong": 414,
      "answered": 2000
    },
    "anthropic/claude-haiku-4.5/control": {
      "conversations": 50,
      "wrong": 454,
      "answered": 1960
    },
    "anthropic/claude-haiku-4.5/break": {
      "conversations": 50,
      "wrong": 535,
      "answered": 2000
    },
    "anthropic/claude-haiku-4.5/pause": {
      "conversations": 50,
      "wrong": 461,
      "answered": 2000
    },
    "google/gemini-3-flash-preview/control": {
      "conversations": 50,
      "wrong": 16,
      "answered": 1998
    },
    "google/gemini-3-flash-preview/break": {
      "conversations": 50,
      "wrong": 28,
      "answered": 1999
    },
    "google/gemini-3-flash-preview/pause": {
      "conversations": 50,
      "wrong": 20,
      "answered": 1999
    },
    "meta-llama/llama-4-maverick/control": {
      "conversations": 50,
      "wrong": 0,
      "answered": 2000
    },
    "meta-llama/llama-4-maverick/break": {
      "conversations": 50,
      "wrong": 11,
      "answered": 2000
    },
    "meta-llama/llama-4-maverick/pause": {
      "conversations": 50,
      "wrong": 19,
      "answered": 2000
    },
    "openai/gpt-5.4-mini/control": {
      "conversations": 50,
      "wrong": 747,
      "answered": 2000
    },
    "openai/gpt-5.4-mini/break": {
      "conversations": 50,
      "wrong": 758,
      "answered": 2000
    },
    "openai/gpt-5.4-mini/pause": {
      "conversations": 50,
      "wrong": 769,
      "answered": 2000
    },
    "x-ai/grok-4.20/control": {
      "conversations": 50,
      "wrong": 437,
      "answered": 2000
    },
    "x-ai/grok-4.20/break": {
      "conversations": 50,
      "wrong": 423,
      "answered": 2000
    },
    "x-ai/grok-4.20/pause": {
      "conversations": 50,
      "wrong": 467,
      "answered": 2000
    }
  },
  "primary": {
    "break vs control": {
      "perModel": {
        "amazon/nova-2-lite-v1": {
          "difference": -0.003,
          "low": -0.006730081454508334,
          "high": -5.592742550320981e-05
        },
        "anthropic/claude-haiku-4.5": {
          "difference": 0.035867346938775524,
          "low": 0.008903144604234123,
          "high": 0.06274551787577817
        },
        "google/gemini-3-flash-preview": {
          "difference": 0.005998995493742868,
          "low": -0.0005652298728081508,
          "high": 0.012885347314337083
        },
        "meta-llama/llama-4-maverick": {
          "difference": 0.0055,
          "low": 0.002407837187875883,
          "high": 0.009822164291298487
        },
        "openai/gpt-5.4-mini": {
          "difference": 0.005500000000000005,
          "low": -0.024507591287275508,
          "high": 0.03549365158265623
        },
        "x-ai/grok-4.20": {
          "difference": -0.007000000000000006,
          "low": -0.03245622120859809,
          "high": 0.018466086294704528
        }
      },
      "cmhChi2": 2.986900544943748,
      "p": 0.08394069205050404
    },
    "pause vs control": {
      "perModel": {
        "amazon/nova-2-lite-v1": {
          "difference": 0.2035,
          "low": 0.1859178359640445,
          "high": 0.2218989148148571
        },
        "anthropic/claude-haiku-4.5": {
          "difference": -0.0011326530612244812,
          "low": -0.027397011801714076,
          "high": 0.025111274214178008
        },
        "google/gemini-3-flash-preview": {
          "difference": 0.001996994493242617,
          "low": -0.004085459788550099,
          "high": 0.008208957935980397
        },
        "meta-llama/llama-4-maverick": {
          "difference": 0.0095,
          "low": 0.0055882055428106165,
          "high": 0.014790501155625134
        },
        "openai/gpt-5.4-mini": {
          "difference": 0.01100000000000001,
          "low": -0.019055976694513324,
          "high": 0.0410280068221167
        },
        "x-ai/grok-4.20": {
          "difference": 0.015000000000000013,
          "low": -0.010926645671021869,
          "high": 0.04090340328690019
        }
      },
      "cmhChi2": 82.89194763281702,
      "p": 8.666349962530464e-20
    }
  },
  "secondary": {
    "amazon/nova-2-lite-v1/control": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 5,
      "errorEarly": 0.007,
      "errorLate": 0.0035
    },
    "amazon/nova-2-lite-v1/break": {
      "unansweredRounds": 0,
      "selfReports": 48,
      "selfReportMedian": 5,
      "errorEarly": 0.002,
      "errorLate": 0.0005
    },
    "amazon/nova-2-lite-v1/pause": {
      "unansweredRounds": 0,
      "selfReports": 48,
      "selfReportMedian": 5,
      "errorEarly": 0.202,
      "errorLate": 0.207,
      "pausedShare": 0.0,
      "pausedInterval": [
        0.0,
        0.07135003417431873
      ],
      "firstPauseRounds": []
    },
    "anthropic/claude-haiku-4.5/control": {
      "unansweredRounds": 8,
      "selfReports": 50,
      "selfReportMedian": 4,
      "errorEarly": 0.25255102040816324,
      "errorLate": 0.2316326530612245
    },
    "anthropic/claude-haiku-4.5/break": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 4,
      "errorEarly": 0.259,
      "errorLate": 0.2675
    },
    "anthropic/claude-haiku-4.5/pause": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 4,
      "errorEarly": 0.2605,
      "errorLate": 0.2305,
      "pausedShare": 0.0,
      "pausedInterval": [
        0.0,
        0.07135003417431873
      ],
      "firstPauseRounds": []
    },
    "google/gemini-3-flash-preview/control": {
      "unansweredRounds": 1,
      "selfReports": 50,
      "selfReportMedian": 4,
      "errorEarly": 0.00963000506842372,
      "errorLate": 0.008008008008008008
    },
    "google/gemini-3-flash-preview/break": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 4,
      "errorEarly": 0.014536340852130326,
      "errorLate": 0.014007003501750876
    },
    "google/gemini-3-flash-preview/pause": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 4,
      "errorEarly": 0.016549648946840523,
      "errorLate": 0.010005002501250625,
      "pausedShare": 0.02,
      "pausedInterval": [
        0.0035391680889764604,
        0.10495686471836953
      ],
      "firstPauseRounds": [
        7
      ]
    },
    "meta-llama/llama-4-maverick/control": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 4,
      "errorEarly": 0.003,
      "errorLate": 0.0
    },
    "meta-llama/llama-4-maverick/break": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 4,
      "errorEarly": 0.01,
      "errorLate": 0.0055
    },
    "meta-llama/llama-4-maverick/pause": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 4,
      "errorEarly": 0.013,
      "errorLate": 0.0095,
      "pausedShare": 0.0,
      "pausedInterval": [
        0.0,
        0.07135003417431873
      ],
      "firstPauseRounds": []
    },
    "openai/gpt-5.4-mini/control": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 5,
      "errorEarly": 0.366,
      "errorLate": 0.3735
    },
    "openai/gpt-5.4-mini/break": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 5,
      "errorEarly": 0.3735,
      "errorLate": 0.379
    },
    "openai/gpt-5.4-mini/pause": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 5,
      "errorEarly": 0.3755,
      "errorLate": 0.3845,
      "pausedShare": 0.0,
      "pausedInterval": [
        0.0,
        0.07135003417431873
      ],
      "firstPauseRounds": []
    },
    "x-ai/grok-4.20/control": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 5,
      "errorEarly": 0.2075,
      "errorLate": 0.2185
    },
    "x-ai/grok-4.20/break": {
      "unansweredRounds": 0,
      "selfReports": 46,
      "selfReportMedian": 4,
      "errorEarly": 0.203,
      "errorLate": 0.2115
    },
    "x-ai/grok-4.20/pause": {
      "unansweredRounds": 0,
      "selfReports": 50,
      "selfReportMedian": 5,
      "errorEarly": 0.205,
      "errorLate": 0.2335,
      "pausedShare": 0.0,
      "pausedInterval": [
        0.0,
        0.07135003417431873
      ],
      "firstPauseRounds": []
    }
  },
  "excludedPilot": true
}