{
  "alt": "Dot chart with two dots per label count. Recalibrated Jev is left of raw Jev from a few hundred labels on. The classifier starts well to the right and comes close to Jev only at the largest samples.",
  "caption": "Further left is cheaper. Both arms use the same labeled complaints from earlier years, and the line marks raw Jev. Each count is every local label used, including the classifier's calibration labels; the complaints being scored are not among them. Whiskers are ranges across draws and not confidence intervals.",
  "data": [
    {
      "conf_high": 11720,
      "conf_low": 4815,
      "cost": 6289,
      "labels": "50 labels",
      "model": "Classifier, fit and calibrated on these labels",
      "n_labels": 50
    },
    {
      "conf_high": 4395,
      "conf_low": 3471,
      "cost": 3899,
      "labels": "50 labels",
      "model": "Jev, recalibrated on these labels",
      "n_labels": 50
    },
    {
      "conf_high": 6643,
      "conf_low": 4417,
      "cost": 5303,
      "labels": "100 labels",
      "model": "Classifier, fit and calibrated on these labels",
      "n_labels": 100
    },
    {
      "conf_high": 5308,
      "conf_low": 3594,
      "cost": 4128,
      "labels": "100 labels",
      "model": "Jev, recalibrated on these labels",
      "n_labels": 100
    },
    {
      "conf_high": 6000,
      "conf_low": 4733,
      "cost": 5125,
      "labels": "200 labels",
      "model": "Classifier, fit and calibrated on these labels",
      "n_labels": 200
    },
    {
      "conf_high": 4556,
      "conf_low": 3471,
      "cost": 3868,
      "labels": "200 labels",
      "model": "Jev, recalibrated on these labels",
      "n_labels": 200
    },
    {
      "conf_high": 5851,
      "conf_low": 4468,
      "cost": 4792,
      "labels": "500 labels",
      "model": "Classifier, fit and calibrated on these labels",
      "n_labels": 500
    },
    {
      "conf_high": 3769,
      "conf_low": 3532,
      "cost": 3613,
      "labels": "500 labels",
      "model": "Jev, recalibrated on these labels",
      "n_labels": 500
    },
    {
      "conf_high": 5187,
      "conf_low": 4272,
      "cost": 4500,
      "labels": "1,000 labels",
      "model": "Classifier, fit and calibrated on these labels",
      "n_labels": 1000
    },
    {
      "conf_high": 4054,
      "conf_low": 3471,
      "cost": 3596,
      "labels": "1,000 labels",
      "model": "Jev, recalibrated on these labels",
      "n_labels": 1000
    },
    {
      "conf_high": 4497,
      "conf_low": 4215,
      "cost": 4326,
      "labels": "2,000 labels",
      "model": "Classifier, fit and calibrated on these labels",
      "n_labels": 2000
    },
    {
      "conf_high": 3839,
      "conf_low": 3471,
      "cost": 3587,
      "labels": "2,000 labels",
      "model": "Jev, recalibrated on these labels",
      "n_labels": 2000
    },
    {
      "conf_high": 4208,
      "conf_low": 4026,
      "cost": 4119,
      "labels": "5,000 labels",
      "model": "Classifier, fit and calibrated on these labels",
      "n_labels": 5000
    },
    {
      "conf_high": 3594,
      "conf_low": 3471,
      "cost": 3528,
      "labels": "5,000 labels",
      "model": "Jev, recalibrated on these labels",
      "n_labels": 5000
    },
    {
      "conf_high": 4145,
      "conf_low": 3889,
      "cost": 3993,
      "labels": "10,000 labels",
      "model": "Classifier, fit and calibrated on these labels",
      "n_labels": 10000
    },
    {
      "conf_high": 3594,
      "conf_low": 3471,
      "cost": 3546,
      "labels": "10,000 labels",
      "model": "Jev, recalibrated on these labels",
      "n_labels": 10000
    },
    {
      "conf_high": 3837,
      "conf_low": 3683,
      "cost": 3742,
      "labels": "30,000 labels",
      "model": "Classifier, fit and calibrated on these labels",
      "n_labels": 30000
    },
    {
      "conf_high": null,
      "conf_low": null,
      "cost": 3542,
      "labels": "30,000 labels",
      "model": "Jev, recalibrated on these labels",
      "n_labels": 30000
    }
  ],
  "id": "cost-by-labels",
  "notes": [
    "Platt scaling in place of isotonic calibration gives the same reading."
  ],
  "source": "CFPB Consumer Complaint Database; labels from August 2023 to 2024; policy scored on 2025 complaints, weighted; engine calls October 2026; assumed costs",
  "spec": {
    "colors": {
      "Classifier, fit and calibrated on these labels": "cat2",
      "Jev, recalibrated on these labels": "cat1"
    },
    "group": "model",
    "hi": "conf_high",
    "lo": "conf_low",
    "reference": 4054,
    "referenceLabel": "Raw Jev",
    "sort": "data",
    "x": "cost",
    "xDomain": [
      3000,
      12000
    ],
    "xFormat": {
      "digits": 0,
      "prefix": "$",
      "thousands": true
    },
    "xLabel": "Expected cost (US dollars per thousand complaints)",
    "y": "labels"
  },
  "subtitle": "Expected routing cost per thousand complaints against local labels used, weighted; whiskers run from the lowest to the highest draw",
  "title": "For the same labels, recalibrating Jev costs less than fitting a classifier at every size tried",
  "type": "dot"
}