{
  "agree_jev": {
    "ci": [
      80.5,
      83.2
    ],
    "ciLevel": 0.95,
    "display": "81.9%",
    "estimand": "top product equals the consumer-selected product",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "percent of complaints",
    "value": 81.8601,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "agree_tfidf": {
    "ci": [
      81.2,
      83.7
    ],
    "ciLevel": 0.95,
    "display": "82.4%",
    "estimand": "top product equals the consumer-selected product",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "percent of complaints",
    "value": 82.4425,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "auc_across_diff_jev_size": {
    "ci": [
      -0.03,
      0.05
    ],
    "ciLevel": 0.95,
    "display": "+0.01",
    "estimand": "jev minus largest diff first",
    "n": 1941,
    "nUnit": "patches",
    "note": "From allocating-review-attention-to-code-changes. ",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "difference in AUC",
    "value": 0.00630002,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "auc_done_jev": {
    "ci": [
      0.55,
      0.83
    ],
    "ciLevel": 0.95,
    "display": "0.71",
    "estimand": "P(done) against the grader's verdict on finish attempts",
    "n": 200,
    "nUnit": "finish attempts",
    "note": "From deciding-when-an-agent-is-finished. ",
    "period": "agent runs and engine calls, October 2026",
    "population": "200 finish attempts by one agent on 40 tasks written for this booklet",
    "source": "constructed-agent-tasks",
    "unit": "area under the ROC curve",
    "value": 0.713257,
    "weights": "unweighted; the mix of tasks and faults describes the test harness"
  },
  "auc_money_jev": {
    "ci": [
      0.881,
      0.908
    ],
    "ciLevel": 0.95,
    "display": "0.895",
    "estimand": "P(money back) against closed with monetary relief",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "area under the ROC curve",
    "value": 0.895398,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "auc_money_product_only": {
    "display": "0.906",
    "estimand": "ranking complaints by their product's relief rate in the training years",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "area under the ROC curve",
    "value": 0.905577,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "auc_money_tfidf": {
    "ci": [
      0.928,
      0.945
    ],
    "ciLevel": 0.95,
    "display": "0.936",
    "estimand": "P(money back) against closed with monetary relief",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "area under the ROC curve",
    "value": 0.936235,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "auc_patch_real": {
    "ci": [
      0.56,
      0.64
    ],
    "ciLevel": 0.95,
    "display": "0.60",
    "estimand": "unresolved ranked above resolved, patches for different issues",
    "n": 1941,
    "nUnit": "patches",
    "note": "From allocating-review-attention-to-code-changes. ",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "area under the ROC curve",
    "value": 0.603585,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "auc_patch_written": {
    "ci": [
      0.69,
      0.89
    ],
    "ciLevel": 0.95,
    "display": "0.79",
    "estimand": "ranks defective patches above acceptable ones",
    "n": 40,
    "nUnit": "patches",
    "note": "From allocating-review-attention-to-code-changes. ",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "area under the ROC curve",
    "value": 0.785,
    "weights": "unweighted; evenly split by construction"
  },
  "auc_reviews_jev": {
    "ci": [
      0.978,
      0.982
    ],
    "ciLevel": 0.95,
    "display": "0.980",
    "estimand": "weighted AUC for dissatisfied (one or two stars)",
    "n": 5000,
    "nUnit": "reviews",
    "note": "From from-evidence-to-typed-questions. ",
    "period": "reviews written 2006 to 2023; engines called October 2026",
    "population": "5,000 sampled coffee reviews in Amazon Reviews 2023, Grocery and Gourmet Food",
    "source": "amazon-reviews-2023",
    "unit": "area under the ROC curve",
    "value": 0.980376,
    "weights": "weighted to the star mix of the coffee reviews (sample_weight)"
  },
  "auc_within_jev": {
    "ci": [
      0.709,
      0.77
    ],
    "ciLevel": 0.95,
    "display": "0.742",
    "estimand": "mean across products of the AUC within each product that has both outcomes",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "area under the ROC curve",
    "value": 0.741876,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "auc_within_tfidf": {
    "ci": [
      0.736,
      0.802
    ],
    "ciLevel": 0.95,
    "display": "0.769",
    "estimand": "mean across products of the AUC within each product that has both outcomes",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "area under the ROC curve",
    "value": 0.768982,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "capture_jev": {
    "ci": [
      53,
      63.4
    ],
    "ciLevel": 0.95,
    "display": "58.2%",
    "estimand": "share caught by fast-tracking the tenth of complaints with the highest P(money back)",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "percent of complaints closed with monetary relief",
    "value": 58.2238,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "capture_tfidf": {
    "ci": [
      68.4,
      77.9
    ],
    "ciLevel": 0.95,
    "display": "73.1%",
    "estimand": "share caught by fast-tracking the tenth of complaints with the highest P(money back)",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "percent of complaints closed with monetary relief",
    "value": 73.085,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "cost_complaints": {
    "display": "$0.05",
    "estimand": "mean billed cost per call, list price",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "US dollars per 1,000 complaints",
    "value": 0.0509582,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "cut_data": {
    "display": "0.38",
    "estimand": "median of five cross-fit cost-minimizing cutoffs",
    "note": "From from-probabilities-to-actions. Range across folds 0.35 to 0.38.",
    "period": "reviews written 2006 to 2023; engines called October 2026",
    "population": "5,000 sampled coffee reviews in Amazon Reviews 2023, Grocery and Gourmet Food",
    "source": "amazon-reviews-2023",
    "unit": "probability of dissatisfaction",
    "value": 0.38,
    "weights": "weighted to the star mix of the coffee reviews (sample_weight)"
  },
  "cut_theory": {
    "display": "0.125",
    "estimand": "cutoff c / (rV)",
    "note": "From from-probabilities-to-actions. ",
    "period": "assumed inputs, October 2026",
    "population": "the coffee roaster's contact decision (assumed inputs)",
    "source": "amazon-reviews-2023",
    "unit": "probability of dissatisfaction",
    "value": 0.125,
    "weights": "not applicable; assumed inputs"
  },
  "lc_capture_1000": {
    "display": "58.8%",
    "estimand": "mean top-tenth capture over refits on 1,000 labeled complaints",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "percent of complaints closed with monetary relief",
    "value": 58.8489,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "lc_net_1000": {
    "display": "$4,466",
    "estimand": "mean net value over refits on 1,000 labeled reviews, at assumed contact costs",
    "n": 5000,
    "nUnit": "reviews",
    "note": "From from-evidence-to-typed-questions. ",
    "period": "reviews written 2006 to 2023; engines called October 2026",
    "population": "5,000 sampled coffee reviews in Amazon Reviews 2023, Grocery and Gourmet Food",
    "source": "amazon-reviews-2023",
    "unit": "US dollars per 1,000 reviews",
    "value": 4465.96,
    "weights": "weighted to the star mix of the coffee reviews (sample_weight)"
  },
  "lc_net_20000": {
    "display": "$5,047",
    "estimand": "mean net value over refits on 20,000 labeled reviews, at assumed contact costs",
    "n": 5000,
    "nUnit": "reviews",
    "note": "From from-evidence-to-typed-questions. ",
    "period": "reviews written 2006 to 2023; engines called October 2026",
    "population": "5,000 sampled coffee reviews in Amazon Reviews 2023, Grocery and Gourmet Food",
    "source": "amazon-reviews-2023",
    "unit": "US dollars per 1,000 reviews",
    "value": 5047.19,
    "weights": "weighted to the star mix of the coffee reviews (sample_weight)"
  },
  "loss_complaints": {
    "ci": [
      3243,
      3709
    ],
    "ciLevel": 0.95,
    "display": "$3,471",
    "estimand": "expected misroute and triage cost",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. Assumed costs; measured error mix against the consumer's product.",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "US dollars per 1,000 complaints",
    "value": 3470.88,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "loss_diff_jev_cal": {
    "ci": [
      -419,
      27
    ],
    "ciLevel": 0.95,
    "display": "−$195",
    "estimand": "jev_cal minus the classifier with a cutoff",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "US dollars per 1,000 complaints",
    "value": -195.364,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "loss_jev_cal": {
    "ci": [
      3243,
      3709
    ],
    "ciLevel": 0.95,
    "display": "$3,471",
    "estimand": "expected misroute and triage cost",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. Assumed costs; measured error mix against the consumer's product.",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "US dollars per 1,000 complaints",
    "value": 3470.88,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "loss_tfidf_cal": {
    "ci": [
      3466,
      3885
    ],
    "ciLevel": 0.95,
    "display": "$3,666",
    "estimand": "expected misroute and triage cost",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. Assumed costs; measured error mix against the consumer's product.",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "US dollars per 1,000 complaints",
    "value": 3666.24,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "lv_clf_200": {
    "display": "$3,257",
    "estimand": "net value at p*, mean across label draws",
    "note": "From from-probabilities-to-actions. Assumed costs. Both arms use the same labeled draws.",
    "period": "reviews written 2006 to 2023; engines called October 2026",
    "population": "the coffee reviews, labels drawn from reviews of other products",
    "source": "amazon-reviews-2023",
    "unit": "US dollars per thousand reviews",
    "value": 3256.7,
    "weights": "weighted to the star mix of the coffee reviews (sample_weight)"
  },
  "lv_cost_clf_500": {
    "display": "$4,792",
    "estimand": "expected misroute and triage cost at the cutoff, mean across label draws",
    "note": "From resolving-and-routing-customer-complaints. Assumed costs. Both arms use the same labeled draws.",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters; labels from August 2023 to 2024 complaints",
    "source": "cfpb-complaints",
    "unit": "US dollars per thousand complaints",
    "value": 4792.08,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "lv_cost_jev_500": {
    "display": "$3,613",
    "estimand": "expected misroute and triage cost at the cutoff, mean across label draws",
    "note": "From resolving-and-routing-customer-complaints. Assumed costs. Both arms use the same labeled draws.",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters; labels from August 2023 to 2024 complaints",
    "source": "cfpb-complaints",
    "unit": "US dollars per thousand complaints",
    "value": 3612.61,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "lv_jev_200": {
    "display": "$5,257",
    "estimand": "net value at p*, mean across label draws",
    "note": "From from-probabilities-to-actions. Assumed costs. Both arms use the same labeled draws.",
    "period": "reviews written 2006 to 2023; engines called October 2026",
    "population": "the coffee reviews, labels drawn from reviews of other products",
    "source": "amazon-reviews-2023",
    "unit": "US dollars per thousand reviews",
    "value": 5256.89,
    "weights": "weighted to the star mix of the coffee reviews (sample_weight)"
  },
  "met_accept": {
    "ci": [
      81,
      93
    ],
    "ciLevel": 0.95,
    "display": "87.5%",
    "estimand": "accepted final state meets every rule-checked criterion",
    "n": 200,
    "nUnit": "runs",
    "note": "From deciding-when-an-agent-is-finished. ",
    "period": "agent runs and engine calls, October 2026",
    "population": "200 runs by one agent on 40 tasks written for this booklet, each policy run to the end of the task",
    "source": "constructed-agent-tasks",
    "unit": "percent of runs",
    "value": 87.5,
    "weights": "unweighted; the mix of tasks and faults describes the test harness"
  },
  "met_check_named": {
    "ci": [
      90.5,
      97
    ],
    "ciLevel": 0.95,
    "display": "94.0%",
    "estimand": "accepted final state meets every rule-checked criterion",
    "n": 200,
    "nUnit": "runs",
    "note": "From deciding-when-an-agent-is-finished. ",
    "period": "agent runs and engine calls, October 2026",
    "population": "200 runs by one agent on 40 tasks written for this booklet, each policy run to the end of the task",
    "source": "constructed-agent-tasks",
    "unit": "percent of runs",
    "value": 94,
    "weights": "unweighted; the mix of tasks and faults describes the test harness"
  },
  "met_jev": {
    "ci": [
      85.5,
      94.5
    ],
    "ciLevel": 0.95,
    "display": "90.0%",
    "estimand": "accepted final state meets every rule-checked criterion",
    "n": 200,
    "nUnit": "runs",
    "note": "From deciding-when-an-agent-is-finished. ",
    "period": "agent runs and engine calls, October 2026",
    "population": "200 runs by one agent on 40 tasks written for this booklet, each policy run to the end of the task",
    "source": "constructed-agent-tasks",
    "unit": "percent of runs",
    "value": 90,
    "weights": "unweighted; the mix of tasks and faults describes the test harness"
  },
  "met_recheck": {
    "ci": [
      85,
      94
    ],
    "ciLevel": 0.95,
    "display": "89.5%",
    "estimand": "accepted final state meets every rule-checked criterion",
    "n": 200,
    "nUnit": "runs",
    "note": "From deciding-when-an-agent-is-finished. ",
    "period": "agent runs and engine calls, October 2026",
    "population": "200 runs by one agent on 40 tasks written for this booklet, each policy run to the end of the task",
    "source": "constructed-agent-tasks",
    "unit": "percent of runs",
    "value": 89.5,
    "weights": "unweighted; the mix of tasks and faults describes the test harness"
  },
  "net_jev": {
    "ci": [
      5004,
      5144
    ],
    "ciLevel": 0.95,
    "display": "$5,076",
    "estimand": "net value of contacts against contacting nobody, at the cutoff p*",
    "n": 5000,
    "nUnit": "reviews",
    "note": "From from-probabilities-to-actions. Assumed costs; measured error mix.",
    "period": "reviews written 2006 to 2023; engines called October 2026",
    "population": "5,000 sampled coffee reviews in Amazon Reviews 2023, Grocery and Gourmet Food",
    "source": "amazon-reviews-2023",
    "unit": "US dollars per 1,000 reviews",
    "value": 5076.08,
    "weights": "weighted to the star mix of the coffee reviews (sample_weight)"
  },
  "net_jev_recal": {
    "ci": [
      5273,
      5393
    ],
    "ciLevel": 0.95,
    "display": "$5,335",
    "estimand": "net value per 1,000 reviews",
    "n": 5000,
    "nUnit": "reviews",
    "note": "From from-probabilities-to-actions. Assumed costs; measured error mix.",
    "period": "reviews written 2006 to 2023; engines called October 2026",
    "population": "5,000 sampled coffee reviews in Amazon Reviews 2023, Grocery and Gourmet Food",
    "source": "amazon-reviews-2023",
    "unit": "US dollars per 1,000 reviews",
    "value": 5335.33,
    "weights": "weighted to the star mix of the coffee reviews (sample_weight)"
  },
  "net_tfidf": {
    "ci": [
      4928,
      5104
    ],
    "ciLevel": 0.95,
    "display": "$5,015",
    "estimand": "net value of contacts against contacting nobody, at the cutoff p*",
    "n": 5000,
    "nUnit": "reviews",
    "note": "From from-probabilities-to-actions. Assumed costs; measured error mix.",
    "period": "reviews written 2006 to 2023; engines called October 2026",
    "population": "5,000 sampled coffee reviews in Amazon Reviews 2023, Grocery and Gourmet Food",
    "source": "amazon-reviews-2023",
    "unit": "US dollars per 1,000 reviews",
    "value": 5015.5,
    "weights": "weighted to the star mix of the coffee reviews (sample_weight)"
  },
  "queue_diff_value_per_minute_cost": {
    "ci": [
      -639,
      -198
    ],
    "ciLevel": 0.95,
    "display": "−425",
    "estimand": "value_per_minute minus reviewing the least certain, cost",
    "n": 10000,
    "nUnit": "complaints",
    "note": "From resolving-and-routing-customer-complaints. ",
    "period": "complaints received in 2025; engines called October 2026",
    "population": "10,000 sampled 2025 CFPB complaints with a narrative of at most 3,000 characters",
    "source": "cfpb-complaints",
    "unit": "per 1,000 complaints",
    "value": -425.056,
    "weights": "weighted to the product mix of those complaints (sample_weight)"
  },
  "retry_jev": {
    "display": "97.4%",
    "estimand": "next action is retry after a transient failure",
    "n": 38,
    "nUnit": "checkpoints",
    "note": "From deciding-when-an-agent-is-finished. ",
    "period": "agent runs and engine calls, October 2026",
    "population": "38 checkpoints after an injected tool failure",
    "source": "constructed-agent-tasks",
    "unit": "percent of checkpoints",
    "value": 97.3684,
    "weights": "unweighted; the mix of tasks and faults describes the test harness"
  },
  "wording_change": {
    "ci": [
      1.4,
      3.9
    ],
    "ciLevel": 0.95,
    "display": "2.5%",
    "estimand": "answer differs between the two versions of the question",
    "n": 1000,
    "nUnit": "reviews",
    "note": "From from-evidence-to-typed-questions. ",
    "period": "reviews written 2006 to 2023; engines called October 2026",
    "population": "1,000 reviews drawn at random from the sample",
    "source": "amazon-reviews-2023",
    "unit": "percent of reviews",
    "value": 2.49195,
    "weights": "weighted to the star mix of the coffee reviews (sample_weight)"
  }
}