{
 "since": "2026-10-06",
 "model": "claude-sonnet-5",
 "versions": {},
 "output_estimate": {
  "formula": "tokens = 0.382 * visible characters + 0.211 * thinking-signature characters -6.8",
  "calibration_replies": 72,
  "median_error": 0.0265,
  "worst_error": 0.1507
 },
 "cases": {
  "gmail-triage": {
   "naoba": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 316,
     "turns": 33,
     "browser_calls": 29,
     "cost_usd": 0.8156,
     "cost_usd_recorded": 0.6865792000000001,
     "output_tokens_estimated": 13197,
     "output_tokens_recorded": 297
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 391,
     "turns": 30,
     "browser_calls": 26,
     "cost_usd": 1.0376,
     "cost_usd_recorded": 0.8158315,
     "output_tokens_estimated": 22444,
     "output_tokens_recorded": 263
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 456,
     "turns": 59,
     "browser_calls": 56,
     "cost_usd": 1.4464,
     "cost_usd_recorded": 1.2610161,
     "output_tokens_estimated": 19317,
     "output_tokens_recorded": 780
    }
   ],
   "builtin": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 413,
     "turns": 113,
     "browser_calls": 117,
     "cost_usd": 2.8225,
     "cost_usd_recorded": 2.6604914,
     "output_tokens_estimated": 18366,
     "output_tokens_recorded": 2168
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 358,
     "turns": 120,
     "browser_calls": 117,
     "cost_usd": 2.9414,
     "cost_usd_recorded": 2.8052541000000004,
     "output_tokens_estimated": 16140,
     "output_tokens_recorded": 2521
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 407,
     "turns": 137,
     "browser_calls": 134,
     "cost_usd": 3.5781,
     "cost_usd_recorded": 3.4199172,
     "output_tokens_estimated": 18729,
     "output_tokens_recorded": 2910
    }
   ]
  },
  "amazon-books": {
   "naoba": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 239,
     "turns": 36,
     "browser_calls": 32,
     "cost_usd": 0.8374,
     "cost_usd_recorded": 0.7317063999999998,
     "output_tokens_estimated": 11098,
     "output_tokens_recorded": 528
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 197,
     "turns": 18,
     "browser_calls": 15,
     "cost_usd": 0.9339,
     "cost_usd_recorded": 0.8229763,
     "output_tokens_estimated": 11340,
     "output_tokens_recorded": 245
    },
    {
     "passed": false,
     "score": 78,
     "failed": [
      {
       "id": "newest_edition_money",
       "reason": "no live breakdown: {\"asin\":\"1098119061\",\"count\":275,\"oneStar\":16,\"price\":null,\"shipping\":null,\"import\":null,\"total\":null}"
      },
      {
       "id": "ai_engineering_money",
       "reason": "no live breakdown: {\"asin\":\"1098166302\",\"count\":990,\"oneStar\":1,\"price\":null,\"shipping\":null,\"import\":null,\"total\":null}"
      }
     ],
     "seconds": 257,
     "turns": 25,
     "browser_calls": 21,
     "cost_usd": 1.1825,
     "cost_usd_recorded": 1.0476801999999998,
     "output_tokens_estimated": 13774,
     "output_tokens_recorded": 290
    }
   ],
   "builtin": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 333,
     "turns": 54,
     "browser_calls": 56,
     "cost_usd": 1.9366,
     "cost_usd_recorded": 1.7529150000000002,
     "output_tokens_estimated": 18996,
     "output_tokens_recorded": 629
    },
    {
     "passed": false,
     "score": 89,
     "failed": [
      {
       "id": "ai_engineering_money",
       "reason": "no live breakdown: {\"asin\":\"1098166302\",\"count\":990,\"oneStar\":1,\"price\":null,\"shipping\":null,\"import\":null,\"total\":null}"
      }
     ],
     "seconds": 407,
     "turns": 83,
     "browser_calls": 80,
     "cost_usd": 2.3608,
     "cost_usd_recorded": 2.214868,
     "output_tokens_estimated": 15856,
     "output_tokens_recorded": 1260
    },
    {
     "passed": false,
     "score": 78,
     "failed": [
      {
       "id": "newest_edition_money",
       "reason": "no live breakdown: {\"asin\":\"1098119061\",\"count\":275,\"oneStar\":16,\"price\":null,\"shipping\":null,\"import\":null,\"total\":null}"
      },
      {
       "id": "ai_engineering_money",
       "reason": "no live breakdown: {\"asin\":\"1098166302\",\"count\":990,\"oneStar\":1,\"price\":null,\"shipping\":null,\"import\":null,\"total\":null}"
      }
     ],
     "seconds": 280,
     "turns": 66,
     "browser_calls": 69,
     "cost_usd": 2.0629,
     "cost_usd_recorded": 1.9244379999999996,
     "output_tokens_estimated": 14938,
     "output_tokens_recorded": 1088
    }
   ]
  },
  "google-search": {
   "naoba": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 340,
     "turns": 45,
     "browser_calls": 41,
     "cost_usd": 1.1486,
     "cost_usd_recorded": 1.0269163999999997,
     "output_tokens_estimated": 12669,
     "output_tokens_recorded": 497
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 224,
     "turns": 36,
     "browser_calls": 32,
     "cost_usd": 0.8924,
     "cost_usd_recorded": 0.8043123999999999,
     "output_tokens_estimated": 9165,
     "output_tokens_recorded": 352
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 150,
     "turns": 21,
     "browser_calls": 18,
     "cost_usd": 0.4535,
     "cost_usd_recorded": 0.4045716,
     "output_tokens_estimated": 5242,
     "output_tokens_recorded": 347
    }
   ],
   "builtin": [
    {
     "passed": false,
     "score": 80,
     "failed": [
      {
       "id": "ticket",
       "reason": "report {\"currency_code\":null,\"amount\":null}, converter [250,222.05]"
      }
     ],
     "seconds": 181,
     "turns": 52,
     "browser_calls": 49,
     "cost_usd": 1.0915,
     "cost_usd_recorded": 1.0050637000000002,
     "output_tokens_estimated": 9565,
     "output_tokens_recorded": 920
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 289,
     "turns": 74,
     "browser_calls": 71,
     "cost_usd": 1.5656,
     "cost_usd_recorded": 1.4789347000000002,
     "output_tokens_estimated": 10007,
     "output_tokens_recorded": 1341
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 135,
     "turns": 43,
     "browser_calls": 48,
     "cost_usd": 0.8442,
     "cost_usd_recorded": 0.8056900000000001,
     "output_tokens_estimated": 4654,
     "output_tokens_recorded": 806
    }
   ]
  },
  "hackernews-thread": {
   "naoba": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 118,
     "turns": 17,
     "browser_calls": 14,
     "cost_usd": 0.4012,
     "cost_usd_recorded": 0.34017680000000006,
     "output_tokens_estimated": 6258,
     "output_tokens_recorded": 161
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 168,
     "turns": 19,
     "browser_calls": 15,
     "cost_usd": 0.4628,
     "cost_usd_recorded": 0.3725475,
     "output_tokens_estimated": 9303,
     "output_tokens_recorded": 275
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 99,
     "turns": 13,
     "browser_calls": 9,
     "cost_usd": 0.3236,
     "cost_usd_recorded": 0.26961689999999994,
     "output_tokens_estimated": 5627,
     "output_tokens_recorded": 226
    }
   ],
   "builtin": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 161,
     "turns": 29,
     "browser_calls": 26,
     "cost_usd": 0.5714,
     "cost_usd_recorded": 0.5040945,
     "output_tokens_estimated": 7039,
     "output_tokens_recorded": 306
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 134,
     "turns": 22,
     "browser_calls": 21,
     "cost_usd": 0.462,
     "cost_usd_recorded": 0.3994735,
     "output_tokens_estimated": 6384,
     "output_tokens_recorded": 133
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 125,
     "turns": 21,
     "browser_calls": 19,
     "cost_usd": 0.4832,
     "cost_usd_recorded": 0.4255237999999999,
     "output_tokens_estimated": 5922,
     "output_tokens_recorded": 151
    }
   ]
  },
  "wikipedia-facts": {
   "naoba": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 106,
     "turns": 16,
     "browser_calls": 12,
     "cost_usd": 0.3913,
     "cost_usd_recorded": 0.33612130000000007,
     "output_tokens_estimated": 5735,
     "output_tokens_recorded": 217
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 105,
     "turns": 16,
     "browser_calls": 12,
     "cost_usd": 0.3603,
     "cost_usd_recorded": 0.30864860000000005,
     "output_tokens_estimated": 5474,
     "output_tokens_recorded": 309
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 152,
     "turns": 24,
     "browser_calls": 20,
     "cost_usd": 0.5001,
     "cost_usd_recorded": 0.4341091,
     "output_tokens_estimated": 6979,
     "output_tokens_recorded": 380
    }
   ],
   "builtin": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 119,
     "turns": 19,
     "browser_calls": 17,
     "cost_usd": 0.4756,
     "cost_usd_recorded": 0.426928,
     "output_tokens_estimated": 5053,
     "output_tokens_recorded": 184
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 125,
     "turns": 24,
     "browser_calls": 17,
     "cost_usd": 0.5183,
     "cost_usd_recorded": 0.46955630000000004,
     "output_tokens_estimated": 5154,
     "output_tokens_recorded": 275
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 112,
     "turns": 21,
     "browser_calls": 16,
     "cost_usd": 0.4545,
     "cost_usd_recorded": 0.40505610000000003,
     "output_tokens_estimated": 5217,
     "output_tokens_recorded": 269
    }
   ]
  },
  "github-issues": {
   "naoba": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 160,
     "turns": 22,
     "browser_calls": 18,
     "cost_usd": 0.5625,
     "cost_usd_recorded": 0.5045207999999999,
     "output_tokens_estimated": 6075,
     "output_tokens_recorded": 276
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 178,
     "turns": 26,
     "browser_calls": 23,
     "cost_usd": 0.5448,
     "cost_usd_recorded": 0.47191610000000006,
     "output_tokens_estimated": 7663,
     "output_tokens_recorded": 375
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 163,
     "turns": 21,
     "browser_calls": 17,
     "cost_usd": 0.4626,
     "cost_usd_recorded": 0.4014997999999999,
     "output_tokens_estimated": 6446,
     "output_tokens_recorded": 339
    }
   ],
   "builtin": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 207,
     "turns": 35,
     "browser_calls": 33,
     "cost_usd": 0.8271,
     "cost_usd_recorded": 0.7586838000000002,
     "output_tokens_estimated": 7355,
     "output_tokens_recorded": 518
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 252,
     "turns": 51,
     "browser_calls": 48,
     "cost_usd": 0.9629,
     "cost_usd_recorded": 0.8977736999999999,
     "output_tokens_estimated": 7248,
     "output_tokens_recorded": 735
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 183,
     "turns": 38,
     "browser_calls": 35,
     "cost_usd": 0.72,
     "cost_usd_recorded": 0.6763032000000002,
     "output_tokens_estimated": 5058,
     "output_tokens_recorded": 685
    }
   ]
  },
  "booking-stay": {
   "naoba": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 217,
     "turns": 19,
     "browser_calls": 15,
     "cost_usd": 0.4469,
     "cost_usd_recorded": 0.3601922,
     "output_tokens_estimated": 8956,
     "output_tokens_recorded": 290
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 202,
     "turns": 23,
     "browser_calls": 20,
     "cost_usd": 0.519,
     "cost_usd_recorded": 0.4394527000000001,
     "output_tokens_estimated": 8274,
     "output_tokens_recorded": 315
    },
    {
     "passed": false,
     "score": 38,
     "failed": [
      {
       "id": "filters_applied",
       "reason": "report 524 sorted by \"Price (lowest first)\", page 437 sorted by Price (lowest first)"
      },
      {
       "id": "three_cheapest",
       "reason": "report [[\"guest-house-fun-and-rest\",66],[\"lifto-hostel\",66],[\"studio-eli-plovdiv\",72]], page [[\"guest-house-fun-and-rest\",66],[\"studio-eli-plovdiv\",72],[\"venis-house-plovdiv\",76],[\"apartament-boiana\",76],[\"cvetkov-guest-house\",77]]"
      },
      {
       "id": "listing_1",
       "reason": "report [66,\"EUR\",false,66,9.2,1], page [66,false,null,9.2,0.6]"
      },
      {
       "id": "listing_2",
       "reason": "report [66,\"EUR\",false,66,8,9.5], page not in the live list"
      },
      {
       "id": "listing_3",
       "reason": "report [72,\"EUR\",true,80,9.7,1.9], page [72,true,80,9.7,2.5]"
      }
     ],
     "seconds": 207,
     "turns": 25,
     "browser_calls": 21,
     "cost_usd": 0.5673,
     "cost_usd_recorded": 0.49072829999999995,
     "output_tokens_estimated": 7989,
     "output_tokens_recorded": 328
    }
   ],
   "builtin": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 324,
     "turns": 107,
     "browser_calls": 104,
     "cost_usd": 2.5916,
     "cost_usd_recorded": 2.4704275000000004,
     "output_tokens_estimated": 13930,
     "output_tokens_recorded": 1815
    },
    {
     "passed": false,
     "score": 38,
     "failed": [
      {
       "id": "filters_applied",
       "reason": "report 528 sorted by \"Price (lowest first)\", page 437 sorted by Price (lowest first)"
      },
      {
       "id": "three_cheapest",
       "reason": "report [[\"guest-house-fun-and-rest\",66],[\"lifto-hostel\",66],[\"studio-eli-plovdiv\",72]], page [[\"guest-house-fun-and-rest\",66],[\"studio-eli-plovdiv\",72],[\"venis-house-plovdiv\",76],[\"apartament-boiana\",76],[\"cvetkov-guest-house\",77]]"
      },
      {
       "id": "listing_1",
       "reason": "report [66,\"EUR\",false,66,9.2,1], page [66,false,null,9.2,0.6]"
      },
      {
       "id": "listing_2",
       "reason": "report [66,\"EUR\",false,66,8,9.5], page not in the live list"
      },
      {
       "id": "listing_3",
       "reason": "report [72,\"EUR\",true,80,9.7,1.9], page [72,true,80,9.7,2.5]"
      }
     ],
     "seconds": 164,
     "turns": 33,
     "browser_calls": 32,
     "cost_usd": 0.7772,
     "cost_usd_recorded": 0.7094262000000001,
     "output_tokens_estimated": 7201,
     "output_tokens_recorded": 428
    },
    {
     "passed": false,
     "score": 75,
     "failed": [
      {
       "id": "filters_applied",
       "reason": "report 280 sorted by \"Price (lowest first)\", page 438 sorted by Price (lowest first)"
      },
      {
       "id": "three_cheapest",
       "reason": "report [[\"guest-house-fun-and-rest\",66],[\"venis-house-plovdiv\",76],[\"cvetkov-guest-house\",77]], page [[\"guest-house-fun-and-rest\",66],[\"studio-eli-plovdiv\",72],[\"venis-house-plovdiv\",76],[\"apartament-boiana\",76],[\"cvetkov-guest-house\",77]]"
      }
     ],
     "seconds": 258,
     "turns": 72,
     "browser_calls": 70,
     "cost_usd": 1.8917,
     "cost_usd_recorded": 1.7960841,
     "output_tokens_estimated": 10680,
     "output_tokens_recorded": 1117
    }
   ]
  },
  "youtube-talk": {
   "naoba": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 275,
     "turns": 26,
     "browser_calls": 22,
     "cost_usd": 0.5357,
     "cost_usd_recorded": 0.44770769999999993,
     "output_tokens_estimated": 9109,
     "output_tokens_recorded": 312
    },
    {
     "passed": false,
     "score": 90,
     "failed": [
      {
       "id": "latest_upload",
       "reason": "report [\"ad7i3NN8cXI\",\"C++26 for Lazy Programmers • Will Briggs & Todd Montgomery • GOTO 2026\",\"2026-09-26\"], page [[\"ad7i3NN8cXI\",\"C++26 for Lazy Programmers • Will Briggs & Todd Montgomery • GOTO 2026\",\"2026-09-25T01:38:39-07:00\",null],[\"mfWRYv2V71k\",\"Thinking in Platforms • Kasper von Grünberg & "
      }
     ],
     "seconds": 230,
     "turns": 25,
     "browser_calls": 21,
     "cost_usd": 0.5426,
     "cost_usd_recorded": 0.45986849999999996,
     "output_tokens_estimated": 8579,
     "output_tokens_recorded": 305
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 328,
     "turns": 36,
     "browser_calls": 33,
     "cost_usd": 0.8171,
     "cost_usd_recorded": 0.7041005000000001,
     "output_tokens_estimated": 12521,
     "output_tokens_recorded": 1224
    }
   ],
   "builtin": [
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 161,
     "turns": 38,
     "browser_calls": 35,
     "cost_usd": 0.7591,
     "cost_usd_recorded": 0.6917796999999999,
     "output_tokens_estimated": 7423,
     "output_tokens_recorded": 694
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 126,
     "turns": 25,
     "browser_calls": 22,
     "cost_usd": 0.4903,
     "cost_usd_recorded": 0.4366554999999999,
     "output_tokens_estimated": 5734,
     "output_tokens_recorded": 368
    },
    {
     "passed": true,
     "score": 100,
     "failed": [],
     "seconds": 116,
     "turns": 29,
     "browser_calls": 26,
     "cost_usd": 0.5888,
     "cost_usd_recorded": 0.5345299,
     "output_tokens_estimated": 5878,
     "output_tokens_recorded": 455
    }
   ]
  }
 }
}