{
  "schemaVersion": 1,
  "kind": "published_metrics_extract",
  "checkedAt": "2026-09-06",
  "scope": "12 localization tasks on public tRPC in one live session; not a competitor benchmark",
  "model": "Sonnet 5",
  "applicationVersion": "0.3.15",
  "currentReleaseNote": "0.3.16 routing hotfix has not received a new paid localization benchmark",
  "proofLedgerRow": 58,
  "sourceSha256": "bbcf146a273d6c9f7d0766d53f8373691fd245c4ba74b24533e711c19d77e9d8",
  "summary": {
    "correct": 12,
    "correctOneMainCall": 12,
    "mainCalls": 12,
    "inputTokens": 144900,
    "costRub": 4.36059,
    "medianAnswerMs": 9439.5,
    "selectorCalls": 0
  },
  "tasks": [
    {
      "id": "batch-link",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 10081,
      "wallMs": 10173.493000000002,
      "mainCalls": 1,
      "inputTokens": 10163,
      "costRub": 0.30585
    },
    {
      "id": "response-resolver",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 10169,
      "wallMs": 10257.438000000002,
      "mainCalls": 1,
      "inputTokens": 14850,
      "costRub": 0.44667
    },
    {
      "id": "batch-options",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 9914,
      "wallMs": 10000.012750000002,
      "mainCalls": 1,
      "inputTokens": 9504,
      "costRub": 0.28617
    },
    {
      "id": "transform-callers",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 10423,
      "wallMs": 10509.474125,
      "mainCalls": 1,
      "inputTokens": 10482,
      "costRub": 0.31656
    },
    {
      "id": "prototype-test",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 9433,
      "wallMs": 9519.954332999994,
      "mainCalls": 1,
      "inputTokens": 11334,
      "costRub": 0.34143
    },
    {
      "id": "router-factory",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 9195,
      "wallMs": 9282.270916000009,
      "mainCalls": 1,
      "inputTokens": 11118,
      "costRub": 0.33462
    },
    {
      "id": "error-status",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 7957,
      "wallMs": 8045.0533339999965,
      "mainCalls": 1,
      "inputTokens": 11829,
      "costRub": 0.35607
    },
    {
      "id": "fetch-resolver-call",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 6693,
      "wallMs": 6790.262084000002,
      "mainCalls": 1,
      "inputTokens": 12209,
      "costRub": 0.36714
    },
    {
      "id": "client-error",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 7397,
      "wallMs": 7479.877458999996,
      "mainCalls": 1,
      "inputTokens": 13246,
      "costRub": 0.39822
    },
    {
      "id": "share-test",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 8117,
      "wallMs": 8203.034125000006,
      "mainCalls": 1,
      "inputTokens": 15557,
      "costRub": 0.46755
    },
    {
      "id": "merge-formatters",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 10537,
      "wallMs": 10619.856417000003,
      "mainCalls": 1,
      "inputTokens": 11503,
      "costRub": 0.34614
    },
    {
      "id": "stream-batch-link",
      "correct": true,
      "correctOneMainCall": true,
      "answerMs": 9446,
      "wallMs": 9531.700833999988,
      "mainCalls": 1,
      "inputTokens": 13105,
      "costRub": 0.39417
    }
  ],
  "limitations": [
    "One public repository, one main model, 12 fixed tasks, one session.",
    "No universal speed, correctness or savings guarantee.",
    "No direct comparison against Cursor, Windsurf or Cline."
  ],
  "longSession": {
    "proofLedgerRow": 9,
    "model": "gpt-oss-20b",
    "turns": 30,
    "inputTokensWithoutEngine": 2418083,
    "inputTokensWithEngine": 357139,
    "correctWithoutEngine": 12,
    "correctWithEngine": 30,
    "limitation": "Separate historical 30-turn experiment. Reduced input tokens do not equal the same reduction in every invoice."
  }
}