{
  "alt": "Dot chart with eight rows and two engines. The whole question and addresses-the-issue sit far right. Security sensitive and public interface sit near the coin-flip line. The equal-weight row falls between.",
  "caption": "Dots far right of the coin-flip line separate defective from acceptable patches. Whether a patch addresses the issue carries most of the signal here, and this set was built with defects of a few kinds, so other work could favor other questions.",
  "data": [
    {
      "auc": 0.785,
      "model": "Jev 1.14.0",
      "signal": "Whole question: would it fail?"
    },
    {
      "auc": 0.662,
      "model": "Jev 1.14.0",
      "signal": "Equal weight on the six questions"
    },
    {
      "auc": 0.877,
      "model": "Jev 1.14.0",
      "signal": "Addresses the issue"
    },
    {
      "auc": 0.535,
      "model": "Jev 1.14.0",
      "signal": "Changes a public interface"
    },
    {
      "auc": 0.499,
      "model": "Jev 1.14.0",
      "signal": "Security sensitive"
    },
    {
      "auc": 0.666,
      "model": "Jev 1.14.0",
      "signal": "Unrelated changes"
    },
    {
      "auc": 0.581,
      "model": "Jev 1.14.0",
      "signal": "Tests added"
    },
    {
      "auc": 0.797,
      "model": "Jev 1.14.0",
      "signal": "Too little context to judge"
    },
    {
      "auc": 0.887,
      "model": "OpenAI Luna",
      "signal": "Whole question: would it fail?"
    },
    {
      "auc": 0.681,
      "model": "OpenAI Luna",
      "signal": "Equal weight on the six questions"
    },
    {
      "auc": 0.874,
      "model": "OpenAI Luna",
      "signal": "Addresses the issue"
    },
    {
      "auc": 0.583,
      "model": "OpenAI Luna",
      "signal": "Changes a public interface"
    },
    {
      "auc": 0.502,
      "model": "OpenAI Luna",
      "signal": "Security sensitive"
    },
    {
      "auc": 0.586,
      "model": "OpenAI Luna",
      "signal": "Unrelated changes"
    },
    {
      "auc": 0.512,
      "model": "OpenAI Luna",
      "signal": "Tests added"
    },
    {
      "auc": 0.764,
      "model": "OpenAI Luna",
      "signal": "Too little context to judge"
    }
  ],
  "id": "which-question-helps",
  "source": "Constructed patches written for this booklet, October 2026; 40 patches on 20 issues; unweighted",
  "spec": {
    "colors": {
      "Claude Haiku adapter": "cat3",
      "Jev 1.14.0": "cat1",
      "OpenAI Luna": "cat2"
    },
    "group": "model",
    "reference": 0.5,
    "referenceLabel": "Coin flip",
    "sort": "data",
    "x": "auc",
    "xDomain": [
      0.3,
      1
    ],
    "xFormat": {
      "digits": 2
    },
    "xLabel": "AUC",
    "y": "signal"
  },
  "subtitle": "AUC of each typed question for finding defective patches, constructed patches, by engine",
  "title": "One question does most of the work, and equal weights dilute it",
  "type": "dot"
}