{
  "alt": "Line chart of defective patches found against reviews done for three models, largest diff first and random order. The model lines rise steeply at first and flatten, the random line rises steadily, and the diff-size line stays near or under the random line.",
  "caption": "Follow a line left to right: each step is one more review. The steeper the start, the more a small budget finds. These patches are evenly split by construction, with far more defects than a real queue would hold.",
  "data": [
    {
      "found": 1,
      "policy": "Jev 1.14.0",
      "reviews": 1
    },
    {
      "found": 2,
      "policy": "Jev 1.14.0",
      "reviews": 2
    },
    {
      "found": 3,
      "policy": "Jev 1.14.0",
      "reviews": 3
    },
    {
      "found": 4,
      "policy": "Jev 1.14.0",
      "reviews": 4
    },
    {
      "found": 5,
      "policy": "Jev 1.14.0",
      "reviews": 5
    },
    {
      "found": 5,
      "policy": "Jev 1.14.0",
      "reviews": 6
    },
    {
      "found": 6,
      "policy": "Jev 1.14.0",
      "reviews": 7
    },
    {
      "found": 7,
      "policy": "Jev 1.14.0",
      "reviews": 8
    },
    {
      "found": 8,
      "policy": "Jev 1.14.0",
      "reviews": 9
    },
    {
      "found": 9,
      "policy": "Jev 1.14.0",
      "reviews": 10
    },
    {
      "found": 10,
      "policy": "Jev 1.14.0",
      "reviews": 11
    },
    {
      "found": 11,
      "policy": "Jev 1.14.0",
      "reviews": 12
    },
    {
      "found": 12,
      "policy": "Jev 1.14.0",
      "reviews": 13
    },
    {
      "found": 12,
      "policy": "Jev 1.14.0",
      "reviews": 14
    },
    {
      "found": 12,
      "policy": "Jev 1.14.0",
      "reviews": 15
    },
    {
      "found": 12,
      "policy": "Jev 1.14.0",
      "reviews": 16
    },
    {
      "found": 13,
      "policy": "Jev 1.14.0",
      "reviews": 17
    },
    {
      "found": 13,
      "policy": "Jev 1.14.0",
      "reviews": 18
    },
    {
      "found": 14,
      "policy": "Jev 1.14.0",
      "reviews": 19
    },
    {
      "found": 14,
      "policy": "Jev 1.14.0",
      "reviews": 20
    },
    {
      "found": 14,
      "policy": "Jev 1.14.0",
      "reviews": 21
    },
    {
      "found": 14,
      "policy": "Jev 1.14.0",
      "reviews": 22
    },
    {
      "found": 14,
      "policy": "Jev 1.14.0",
      "reviews": 23
    },
    {
      "found": 15,
      "policy": "Jev 1.14.0",
      "reviews": 24
    },
    {
      "found": 16,
      "policy": "Jev 1.14.0",
      "reviews": 25
    },
    {
      "found": 16,
      "policy": "Jev 1.14.0",
      "reviews": 26
    },
    {
      "found": 17,
      "policy": "Jev 1.14.0",
      "reviews": 27
    },
    {
      "found": 17,
      "policy": "Jev 1.14.0",
      "reviews": 28
    },
    {
      "found": 17,
      "policy": "Jev 1.14.0",
      "reviews": 29
    },
    {
      "found": 17,
      "policy": "Jev 1.14.0",
      "reviews": 30
    },
    {
      "found": 17,
      "policy": "Jev 1.14.0",
      "reviews": 31
    },
    {
      "found": 18,
      "policy": "Jev 1.14.0",
      "reviews": 32
    },
    {
      "found": 18,
      "policy": "Jev 1.14.0",
      "reviews": 33
    },
    {
      "found": 19,
      "policy": "Jev 1.14.0",
      "reviews": 34
    },
    {
      "found": 19,
      "policy": "Jev 1.14.0",
      "reviews": 35
    },
    {
      "found": 19,
      "policy": "Jev 1.14.0",
      "reviews": 36
    },
    {
      "found": 19,
      "policy": "Jev 1.14.0",
      "reviews": 37
    },
    {
      "found": 20,
      "policy": "Jev 1.14.0",
      "reviews": 38
    },
    {
      "found": 20,
      "policy": "Jev 1.14.0",
      "reviews": 39
    },
    {
      "found": 20,
      "policy": "Jev 1.14.0",
      "reviews": 40
    },
    {
      "found": 1,
      "policy": "OpenAI Luna",
      "reviews": 1
    },
    {
      "found": 2,
      "policy": "OpenAI Luna",
      "reviews": 2
    },
    {
      "found": 3,
      "policy": "OpenAI Luna",
      "reviews": 3
    },
    {
      "found": 4,
      "policy": "OpenAI Luna",
      "reviews": 4
    },
    {
      "found": 5,
      "policy": "OpenAI Luna",
      "reviews": 5
    },
    {
      "found": 6,
      "policy": "OpenAI Luna",
      "reviews": 6
    },
    {
      "found": 7,
      "policy": "OpenAI Luna",
      "reviews": 7
    },
    {
      "found": 8,
      "policy": "OpenAI Luna",
      "reviews": 8
    },
    {
      "found": 9,
      "policy": "OpenAI Luna",
      "reviews": 9
    },
    {
      "found": 10,
      "policy": "OpenAI Luna",
      "reviews": 10
    },
    {
      "found": 11,
      "policy": "OpenAI Luna",
      "reviews": 11
    },
    {
      "found": 12,
      "policy": "OpenAI Luna",
      "reviews": 12
    },
    {
      "found": 13,
      "policy": "OpenAI Luna",
      "reviews": 13
    },
    {
      "found": 14,
      "policy": "OpenAI Luna",
      "reviews": 14
    },
    {
      "found": 14,
      "policy": "OpenAI Luna",
      "reviews": 15
    },
    {
      "found": 15,
      "policy": "OpenAI Luna",
      "reviews": 16
    },
    {
      "found": 16,
      "policy": "OpenAI Luna",
      "reviews": 17
    },
    {
      "found": 16,
      "policy": "OpenAI Luna",
      "reviews": 18
    },
    {
      "found": 16,
      "policy": "OpenAI Luna",
      "reviews": 19
    },
    {
      "found": 17,
      "policy": "OpenAI Luna",
      "reviews": 20
    },
    {
      "found": 17,
      "policy": "OpenAI Luna",
      "reviews": 21
    },
    {
      "found": 17,
      "policy": "OpenAI Luna",
      "reviews": 22
    },
    {
      "found": 17,
      "policy": "OpenAI Luna",
      "reviews": 23
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 24
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 25
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 26
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 27
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 28
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 29
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 30
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 31
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 32
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 33
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 34
    },
    {
      "found": 18,
      "policy": "OpenAI Luna",
      "reviews": 35
    },
    {
      "found": 19,
      "policy": "OpenAI Luna",
      "reviews": 36
    },
    {
      "found": 20,
      "policy": "OpenAI Luna",
      "reviews": 37
    },
    {
      "found": 20,
      "policy": "OpenAI Luna",
      "reviews": 38
    },
    {
      "found": 20,
      "policy": "OpenAI Luna",
      "reviews": 39
    },
    {
      "found": 20,
      "policy": "OpenAI Luna",
      "reviews": 40
    },
    {
      "found": 1,
      "policy": "Claude Haiku adapter",
      "reviews": 1
    },
    {
      "found": 2,
      "policy": "Claude Haiku adapter",
      "reviews": 2
    },
    {
      "found": 3,
      "policy": "Claude Haiku adapter",
      "reviews": 3
    },
    {
      "found": 4,
      "policy": "Claude Haiku adapter",
      "reviews": 4
    },
    {
      "found": 5,
      "policy": "Claude Haiku adapter",
      "reviews": 5
    },
    {
      "found": 6,
      "policy": "Claude Haiku adapter",
      "reviews": 6
    },
    {
      "found": 7,
      "policy": "Claude Haiku adapter",
      "reviews": 7
    },
    {
      "found": 8,
      "policy": "Claude Haiku adapter",
      "reviews": 8
    },
    {
      "found": 9,
      "policy": "Claude Haiku adapter",
      "reviews": 9
    },
    {
      "found": 10,
      "policy": "Claude Haiku adapter",
      "reviews": 10
    },
    {
      "found": 10,
      "policy": "Claude Haiku adapter",
      "reviews": 11
    },
    {
      "found": 10,
      "policy": "Claude Haiku adapter",
      "reviews": 12
    },
    {
      "found": 11,
      "policy": "Claude Haiku adapter",
      "reviews": 13
    },
    {
      "found": 12,
      "policy": "Claude Haiku adapter",
      "reviews": 14
    },
    {
      "found": 12,
      "policy": "Claude Haiku adapter",
      "reviews": 15
    },
    {
      "found": 13,
      "policy": "Claude Haiku adapter",
      "reviews": 16
    },
    {
      "found": 14,
      "policy": "Claude Haiku adapter",
      "reviews": 17
    },
    {
      "found": 14,
      "policy": "Claude Haiku adapter",
      "reviews": 18
    },
    {
      "found": 14,
      "policy": "Claude Haiku adapter",
      "reviews": 19
    },
    {
      "found": 15,
      "policy": "Claude Haiku adapter",
      "reviews": 20
    },
    {
      "found": 15,
      "policy": "Claude Haiku adapter",
      "reviews": 21
    },
    {
      "found": 15,
      "policy": "Claude Haiku adapter",
      "reviews": 22
    },
    {
      "found": 15,
      "policy": "Claude Haiku adapter",
      "reviews": 23
    },
    {
      "found": 15,
      "policy": "Claude Haiku adapter",
      "reviews": 24
    },
    {
      "found": 15,
      "policy": "Claude Haiku adapter",
      "reviews": 25
    },
    {
      "found": 16,
      "policy": "Claude Haiku adapter",
      "reviews": 26
    },
    {
      "found": 16,
      "policy": "Claude Haiku adapter",
      "reviews": 27
    },
    {
      "found": 17,
      "policy": "Claude Haiku adapter",
      "reviews": 28
    },
    {
      "found": 18,
      "policy": "Claude Haiku adapter",
      "reviews": 29
    },
    {
      "found": 18,
      "policy": "Claude Haiku adapter",
      "reviews": 30
    },
    {
      "found": 18,
      "policy": "Claude Haiku adapter",
      "reviews": 31
    },
    {
      "found": 18,
      "policy": "Claude Haiku adapter",
      "reviews": 32
    },
    {
      "found": 18,
      "policy": "Claude Haiku adapter",
      "reviews": 33
    },
    {
      "found": 18,
      "policy": "Claude Haiku adapter",
      "reviews": 34
    },
    {
      "found": 18,
      "policy": "Claude Haiku adapter",
      "reviews": 35
    },
    {
      "found": 18,
      "policy": "Claude Haiku adapter",
      "reviews": 36
    },
    {
      "found": 19,
      "policy": "Claude Haiku adapter",
      "reviews": 37
    },
    {
      "found": 19,
      "policy": "Claude Haiku adapter",
      "reviews": 38
    },
    {
      "found": 20,
      "policy": "Claude Haiku adapter",
      "reviews": 39
    },
    {
      "found": 20,
      "policy": "Claude Haiku adapter",
      "reviews": 40
    },
    {
      "found": 0,
      "policy": "Largest diff first",
      "reviews": 1
    },
    {
      "found": 0,
      "policy": "Largest diff first",
      "reviews": 2
    },
    {
      "found": 1,
      "policy": "Largest diff first",
      "reviews": 3
    },
    {
      "found": 2,
      "policy": "Largest diff first",
      "reviews": 4
    },
    {
      "found": 2,
      "policy": "Largest diff first",
      "reviews": 5
    },
    {
      "found": 2,
      "policy": "Largest diff first",
      "reviews": 6
    },
    {
      "found": 3,
      "policy": "Largest diff first",
      "reviews": 7
    },
    {
      "found": 3,
      "policy": "Largest diff first",
      "reviews": 8
    },
    {
      "found": 4,
      "policy": "Largest diff first",
      "reviews": 9
    },
    {
      "found": 5,
      "policy": "Largest diff first",
      "reviews": 10
    },
    {
      "found": 5,
      "policy": "Largest diff first",
      "reviews": 11
    },
    {
      "found": 6,
      "policy": "Largest diff first",
      "reviews": 12
    },
    {
      "found": 7,
      "policy": "Largest diff first",
      "reviews": 13
    },
    {
      "found": 7,
      "policy": "Largest diff first",
      "reviews": 14
    },
    {
      "found": 8,
      "policy": "Largest diff first",
      "reviews": 15
    },
    {
      "found": 8,
      "policy": "Largest diff first",
      "reviews": 16
    },
    {
      "found": 8,
      "policy": "Largest diff first",
      "reviews": 17
    },
    {
      "found": 8,
      "policy": "Largest diff first",
      "reviews": 18
    },
    {
      "found": 9,
      "policy": "Largest diff first",
      "reviews": 19
    },
    {
      "found": 9,
      "policy": "Largest diff first",
      "reviews": 20
    },
    {
      "found": 10,
      "policy": "Largest diff first",
      "reviews": 21
    },
    {
      "found": 10,
      "policy": "Largest diff first",
      "reviews": 22
    },
    {
      "found": 10,
      "policy": "Largest diff first",
      "reviews": 23
    },
    {
      "found": 11,
      "policy": "Largest diff first",
      "reviews": 24
    },
    {
      "found": 12,
      "policy": "Largest diff first",
      "reviews": 25
    },
    {
      "found": 12,
      "policy": "Largest diff first",
      "reviews": 26
    },
    {
      "found": 12,
      "policy": "Largest diff first",
      "reviews": 27
    },
    {
      "found": 13,
      "policy": "Largest diff first",
      "reviews": 28
    },
    {
      "found": 13,
      "policy": "Largest diff first",
      "reviews": 29
    },
    {
      "found": 14,
      "policy": "Largest diff first",
      "reviews": 30
    },
    {
      "found": 15,
      "policy": "Largest diff first",
      "reviews": 31
    },
    {
      "found": 15,
      "policy": "Largest diff first",
      "reviews": 32
    },
    {
      "found": 16,
      "policy": "Largest diff first",
      "reviews": 33
    },
    {
      "found": 17,
      "policy": "Largest diff first",
      "reviews": 34
    },
    {
      "found": 17,
      "policy": "Largest diff first",
      "reviews": 35
    },
    {
      "found": 18,
      "policy": "Largest diff first",
      "reviews": 36
    },
    {
      "found": 19,
      "policy": "Largest diff first",
      "reviews": 37
    },
    {
      "found": 19,
      "policy": "Largest diff first",
      "reviews": 38
    },
    {
      "found": 20,
      "policy": "Largest diff first",
      "reviews": 39
    },
    {
      "found": 20,
      "policy": "Largest diff first",
      "reviews": 40
    },
    {
      "found": 0.5,
      "policy": "Random order (average)",
      "reviews": 1
    },
    {
      "found": 1,
      "policy": "Random order (average)",
      "reviews": 2
    },
    {
      "found": 1.5,
      "policy": "Random order (average)",
      "reviews": 3
    },
    {
      "found": 2,
      "policy": "Random order (average)",
      "reviews": 4
    },
    {
      "found": 2.5,
      "policy": "Random order (average)",
      "reviews": 5
    },
    {
      "found": 3,
      "policy": "Random order (average)",
      "reviews": 6
    },
    {
      "found": 3.5,
      "policy": "Random order (average)",
      "reviews": 7
    },
    {
      "found": 4,
      "policy": "Random order (average)",
      "reviews": 8
    },
    {
      "found": 4.5,
      "policy": "Random order (average)",
      "reviews": 9
    },
    {
      "found": 5,
      "policy": "Random order (average)",
      "reviews": 10
    },
    {
      "found": 5.5,
      "policy": "Random order (average)",
      "reviews": 11
    },
    {
      "found": 6,
      "policy": "Random order (average)",
      "reviews": 12
    },
    {
      "found": 6.5,
      "policy": "Random order (average)",
      "reviews": 13
    },
    {
      "found": 7,
      "policy": "Random order (average)",
      "reviews": 14
    },
    {
      "found": 7.5,
      "policy": "Random order (average)",
      "reviews": 15
    },
    {
      "found": 8,
      "policy": "Random order (average)",
      "reviews": 16
    },
    {
      "found": 8.5,
      "policy": "Random order (average)",
      "reviews": 17
    },
    {
      "found": 9,
      "policy": "Random order (average)",
      "reviews": 18
    },
    {
      "found": 9.5,
      "policy": "Random order (average)",
      "reviews": 19
    },
    {
      "found": 10,
      "policy": "Random order (average)",
      "reviews": 20
    },
    {
      "found": 10.5,
      "policy": "Random order (average)",
      "reviews": 21
    },
    {
      "found": 11,
      "policy": "Random order (average)",
      "reviews": 22
    },
    {
      "found": 11.5,
      "policy": "Random order (average)",
      "reviews": 23
    },
    {
      "found": 12,
      "policy": "Random order (average)",
      "reviews": 24
    },
    {
      "found": 12.5,
      "policy": "Random order (average)",
      "reviews": 25
    },
    {
      "found": 13,
      "policy": "Random order (average)",
      "reviews": 26
    },
    {
      "found": 13.5,
      "policy": "Random order (average)",
      "reviews": 27
    },
    {
      "found": 14,
      "policy": "Random order (average)",
      "reviews": 28
    },
    {
      "found": 14.5,
      "policy": "Random order (average)",
      "reviews": 29
    },
    {
      "found": 15,
      "policy": "Random order (average)",
      "reviews": 30
    },
    {
      "found": 15.5,
      "policy": "Random order (average)",
      "reviews": 31
    },
    {
      "found": 16,
      "policy": "Random order (average)",
      "reviews": 32
    },
    {
      "found": 16.5,
      "policy": "Random order (average)",
      "reviews": 33
    },
    {
      "found": 17,
      "policy": "Random order (average)",
      "reviews": 34
    },
    {
      "found": 17.5,
      "policy": "Random order (average)",
      "reviews": 35
    },
    {
      "found": 18,
      "policy": "Random order (average)",
      "reviews": 36
    },
    {
      "found": 18.5,
      "policy": "Random order (average)",
      "reviews": 37
    },
    {
      "found": 19,
      "policy": "Random order (average)",
      "reviews": 38
    },
    {
      "found": 19.5,
      "policy": "Random order (average)",
      "reviews": 39
    },
    {
      "found": 20,
      "policy": "Random order (average)",
      "reviews": 40
    }
  ],
  "id": "defects-found-by-review",
  "notes": [
    "AUC on this set: Luna {fact:auc_con_luna}, Jev {fact:auc_con_jev}, Haiku adapter {fact:auc_con_adapter}, largest diff first {fact:auc_con_size}."
  ],
  "source": "Constructed patches written for this booklet, October 2026; 40 patches on 20 issues; unweighted",
  "spec": {
    "colors": {
      "Claude Haiku adapter": "cat3",
      "Jev 1.14.0": "cat1",
      "Largest diff first": "highlight",
      "OpenAI Luna": "cat2",
      "Random order (average)": "muted"
    },
    "dashed": [
      "Random order (average)"
    ],
    "series": "policy",
    "x": "reviews",
    "xDomain": [
      1,
      40
    ],
    "xLabel": "Patches reviewed",
    "y": "found",
    "yDomain": [
      0,
      20
    ],
    "yFormat": {
      "digits": 0
    },
    "yLabel": "Defective patches found"
  },
  "subtitle": "Defective patches among the first reviews in each order, constructed patches, if each review reveals the defect",
  "title": "Ranking by the models finds the defects much sooner than ranking by diff size",
  "type": "line"
}