{
  "alt": "Dot chart with one row per engine and up to two dots per row. The as-returned dots for Jev, Luna and the Claude Haiku adapter are far right of the TF-IDF classifier's dot. The recalibrated dots for Jev and Luna sit near zero, left of the classifier.",
  "caption": "Compare each engine's two dots: as returned, Jev, Luna and the adapter sit well right of the classifier; after recalibration on other folds of the sample, Jev and Luna fall close to zero. Calibration error is the weighted gap between the stated probability and the observed share with one or two stars. Whiskers are 95% intervals from a bootstrap that resamples products.",
  "data": [
    {
      "conf_high": 0.0794,
      "conf_low": 0.0713,
      "engine": "Jev 1.14.0",
      "estimate": 0.0752,
      "n": 5000,
      "version": "As returned"
    },
    {
      "conf_high": 0.0062,
      "conf_low": 0.0014,
      "engine": "Jev 1.14.0",
      "estimate": 0.0017,
      "n": 5000,
      "version": "Recalibrated"
    },
    {
      "conf_high": 0.0639,
      "conf_low": 0.0555,
      "engine": "OpenAI Luna",
      "estimate": 0.0596,
      "n": 5000,
      "version": "As returned"
    },
    {
      "conf_high": 0.0064,
      "conf_low": 0.0014,
      "engine": "OpenAI Luna",
      "estimate": 0.0016,
      "n": 5000,
      "version": "Recalibrated"
    },
    {
      "conf_high": 0.0735,
      "conf_low": 0.0657,
      "engine": "Claude Haiku adapter",
      "estimate": 0.0694,
      "n": 5000,
      "version": "As returned"
    },
    {
      "conf_high": 0.0162,
      "conf_low": 0.0068,
      "engine": "TF-IDF + logistic",
      "estimate": 0.0106,
      "n": 5000,
      "version": "As returned"
    }
  ],
  "id": "calibration-error",
  "notes": [
    "The figure recalibrates only the two engines that return a probability distribution."
  ],
  "source": "Amazon Reviews 2023, coffee products; 5,000 sampled reviews, weighted; engine calls October 2026",
  "spec": {
    "colors": {
      "As returned": "highlight",
      "Recalibrated": "cat1"
    },
    "group": "version",
    "hi": "conf_high",
    "lo": "conf_low",
    "n": "n",
    "sort": "data",
    "x": "estimate",
    "xDomain": [
      0,
      0.1
    ],
    "xFormat": {
      "digits": 3
    },
    "xLabel": "Expected calibration error (lower is better)",
    "y": "engine"
  },
  "subtitle": "Expected calibration error of the probability of dissatisfaction, as returned and after recalibration, by engine, weighted",
  "title": "The engines' probabilities are off by more than the classifier's until they are recalibrated",
  "type": "dot",
  "width": "body"
}