{
 "generatedAt": "2026-10-11T00:21:26.483Z",
 "live": "c2026.10.20",
 "sets": {
  "tuned": {
   "label": "Development set",
   "tasks": 50,
   "description": "50 tasks the compiler was developed against. Gains here are optimistic: the rules were found on these tasks."
  },
  "holdout-a": {
   "label": "Held-out A",
   "tasks": 24,
   "description": "24 tasks written by independent authors with no access to the compiler. Used once as held out; its failures then shaped c2026.10.16."
  },
  "holdout-b": {
   "label": "Held-out B",
   "tasks": 24,
   "description": "24 more independent tasks, including four precise specs as controls. Seen after c2026.10.16."
  },
  "holdout-c": {
   "label": "Held-out C",
   "tasks": 24,
   "description": "24 independent tasks, half in larger repositories (25–45 files) with misleading names, where finding the right place is the hard part."
  }
 },
 "versions": [
  {
   "id": "c2026.10.12",
   "date": "2026-10-09",
   "cli": "0.3.1",
   "status": "shipped",
   "title": "Repository-aware localization",
   "summary": "The CLI ranks the files a request is about with a local content search, test pairing and co-change history. Names only leave the machine.",
   "changes": [
    "BM25F ranking over local git grep",
    "Test files paired with source files",
    "Co-change from the last 300 commits",
    "Turkish to English software-noun glossary"
   ],
   "metrics": [
    {
     "label": "Right file in the top 8 (SWE-bench Verified, 60 tasks)",
     "before": 25.4,
     "after": 72.2,
     "unit": "%"
    },
    {
     "label": "Right file first",
     "before": 0.8,
     "after": 36.3,
     "unit": "%"
    }
   ],
   "studies": []
  },
  {
   "id": "c2026.10.13",
   "date": "2026-10-09",
   "cli": "0.4.0",
   "status": "shipped",
   "title": "Function and class names",
   "summary": "The CLI sends the names of functions and classes in the likely files, so the brief can point at where to start reading.",
   "changes": [
    "Function and class names per likely file (opt-out)",
    "Names annotate the file hints, never the requirements"
   ],
   "metrics": [
    {
     "label": "Judge score, 48 coding requests",
     "before": 4.34,
     "after": 4.49,
     "unit": "/5"
    },
    {
     "label": "Right function named when the file is found",
     "before": null,
     "after": 57.4,
     "unit": "%"
    }
   ],
   "studies": []
  },
  {
   "id": "c2026.10.14",
   "date": "2026-10-09",
   "cli": "0.4.0",
   "status": "measured",
   "title": "First real-agent study",
   "summary": "The first time real coding agents ran on raw requests and on v8 briefs, graded by hidden tests. The brief lost: agents treated it as the whole spec and stopped reading the repository.",
   "changes": [
    "50-task benchmark with hidden tests",
    "Claude Code runs isolated from the operator's setup"
   ],
   "studies": [
    {
     "id": "2026-10-09T14-30-52",
     "set": "tuned",
     "setLabel": "Development set",
     "agent": "Claude Code",
     "model": "Sonnet",
     "note": null,
     "rerun": false,
     "tasks": 50,
     "repeats": 2,
     "raw": {
      "runs": 100,
      "resolve": 0.88,
      "turns": 10,
      "cost": 0.0673,
      "seconds": 26.1,
      "outOfScope": 0.58,
      "errors": 0
     },
     "v8": {
      "runs": 100,
      "resolve": 0.75,
      "turns": 6.2,
      "cost": 0.068,
      "seconds": 26.2,
      "outOfScope": 0.51,
      "errors": 0
     },
     "tally": {
      "win": 3,
      "tie": 37,
      "loss": 10
     },
     "categories": {
      "feature": {
       "tasks": 12,
       "raw": 0.833,
       "v8": 0.75
      },
      "undefined": {
       "tasks": 6,
       "raw": 0.917,
       "v8": 0.667
      },
      "bugfix": {
       "tasks": 21,
       "raw": 0.833,
       "v8": 0.69
      },
      "refactor": {
       "tasks": 6,
       "raw": 1,
       "v8": 1
      },
      "scope": {
       "tasks": 5,
       "raw": 1,
       "v8": 0.8
      }
     },
     "results": [
      {
       "id": "amb-cli-remove-command",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-invoice-csv-export",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-orders-pagination",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-payment-retry",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-user-soft-delete",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-validator-iban",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-webhook-refund",
       "category": "feature",
       "language": "js",
       "raw": 0,
       "v8": 0,
       "outcome": "tie"
      },
      {
       "id": "ambiguous-sort-users",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "js-bugfix-slugify",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-contacts-import-crash",
       "category": "bugfix",
       "language": "js",
       "raw": 0,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "jsbug-email-double-escape",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-export-empty-csv",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-invoice-rounding",
       "category": "bugfix",
       "language": "js",
       "raw": 0,
       "v8": 0,
       "outcome": "tie"
      },
      {
       "id": "jsbug-notification-duplicates",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-orders-pagination",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "jsbug-price-cache-ttl",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "jsbug-reminder-due-date",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-search-accents",
       "category": "bugfix",
       "language": "js",
       "raw": 0.5,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "jsbug-semver-latest",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 0.5,
       "outcome": "loss"
      },
      {
       "id": "jsbug-warehouse-duplicate-sku",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "node-cli-flag",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-conv-deactivate-user",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 0.5,
       "outcome": "loss"
      },
      {
       "id": "py-conv-list-endpoint",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-conv-refund",
       "category": "feature",
       "language": "py",
       "raw": 0,
       "v8": 0.5,
       "outcome": "win"
      },
      {
       "id": "py-pytest-feature",
       "raw": 0.5,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "py-tb-bool-parse",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-tb-config-merge",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-tb-money-rounding",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "py-tb-sku-lookup",
       "category": "bugfix",
       "language": "py",
       "raw": 0,
       "v8": 0,
       "outcome": "tie"
      },
      {
       "id": "py-tb-tz-aware",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-vague-digest-missing",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-vague-report-totals",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-vague-search-accents",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-js-move-helper",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-js-rename-money-format",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-js-signature-options",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-py-extract-shared",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-py-rename-config-key",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-py-split-module",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "scope-js-generated-routes",
       "category": "scope",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "scope-js-legacy-csv",
       "category": "scope",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "scope-js-vendor-date",
       "category": "scope",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "scope-py-deprecated-importer",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "scope-py-logging-noise",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "scope-trap-logger",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "tr-coupon-shipping",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "tr-product-category-filter",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "tr-report-timezone",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "tr-user-pagination",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "ts-rename-config",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      }
     ]
    }
   ],
   "metrics": []
  },
  {
   "id": "c2026.10.15",
   "date": "2026-10-10",
   "cli": "0.5.0",
   "status": "shipped",
   "title": "Read the repository first",
   "summary": "Every brief says it was written without reading the code and puts the repository's docs in done. The CLI's context budget went from 300 ms to 1200 ms after we found the content search rarely finished in time on Windows.",
   "changes": [
    "Read-first line naming the matching docs",
    "Symptom numbers never become limits",
    "Unconfirmed file hints, in any wording",
    "A take-a-look defect report is a fix",
    "Context budget 300 ms to 1200 ms"
   ],
   "metrics": [
    {
     "label": "Right file in the top 8, cold machine",
     "before": 37.9,
     "after": 73.9,
     "unit": "%"
    }
   ],
   "studies": [
    {
     "id": "2026-10-10T15-00-00",
     "set": "tuned",
     "setLabel": "Development set",
     "agent": "Claude Code",
     "model": "Sonnet",
     "note": "v8 condition rerun on the same raw runs",
     "rerun": true,
     "tasks": 50,
     "repeats": 2,
     "raw": {
      "runs": 100,
      "resolve": 0.88,
      "turns": 10,
      "cost": 0.0673,
      "seconds": 26.1,
      "outOfScope": 0.58,
      "errors": 0
     },
     "v8": {
      "runs": 100,
      "resolve": 0.97,
      "turns": 7.4,
      "cost": 0.0708,
      "seconds": 25.9,
      "outOfScope": 0.57,
      "errors": 0
     },
     "tally": {
      "win": 6,
      "tie": 43,
      "loss": 1
     },
     "categories": {
      "feature": {
       "tasks": 12,
       "raw": 0.833,
       "v8": 0.958
      },
      "undefined": {
       "tasks": 6,
       "raw": 0.917,
       "v8": 1
      },
      "bugfix": {
       "tasks": 21,
       "raw": 0.833,
       "v8": 0.952
      },
      "refactor": {
       "tasks": 6,
       "raw": 1,
       "v8": 1
      },
      "scope": {
       "tasks": 5,
       "raw": 1,
       "v8": 1
      }
     },
     "results": [
      {
       "id": "amb-cli-remove-command",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-invoice-csv-export",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-orders-pagination",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-payment-retry",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-user-soft-delete",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-validator-iban",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "amb-webhook-refund",
       "category": "feature",
       "language": "js",
       "raw": 0,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "ambiguous-sort-users",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "js-bugfix-slugify",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-contacts-import-crash",
       "category": "bugfix",
       "language": "js",
       "raw": 0,
       "v8": 0,
       "outcome": "tie"
      },
      {
       "id": "jsbug-email-double-escape",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-export-empty-csv",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-invoice-rounding",
       "category": "bugfix",
       "language": "js",
       "raw": 0,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "jsbug-notification-duplicates",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-orders-pagination",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-price-cache-ttl",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-reminder-due-date",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-search-accents",
       "category": "bugfix",
       "language": "js",
       "raw": 0.5,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "jsbug-semver-latest",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "jsbug-warehouse-duplicate-sku",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "node-cli-flag",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-conv-deactivate-user",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-conv-list-endpoint",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-conv-refund",
       "category": "feature",
       "language": "py",
       "raw": 0,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "py-pytest-feature",
       "raw": 0.5,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "py-tb-bool-parse",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-tb-config-merge",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-tb-money-rounding",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-tb-sku-lookup",
       "category": "bugfix",
       "language": "py",
       "raw": 0,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "py-tb-tz-aware",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-vague-digest-missing",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-vague-report-totals",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "py-vague-search-accents",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-js-move-helper",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-js-rename-money-format",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-js-signature-options",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-py-extract-shared",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-py-rename-config-key",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ref-py-split-module",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "scope-js-generated-routes",
       "category": "scope",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "scope-js-legacy-csv",
       "category": "scope",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "scope-js-vendor-date",
       "category": "scope",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "scope-py-deprecated-importer",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "scope-py-logging-noise",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "scope-trap-logger",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "tr-coupon-shipping",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "tr-product-category-filter",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "tr-report-timezone",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "tr-user-pagination",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 0.5,
       "outcome": "loss"
      },
      {
       "id": "ts-rename-config",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      }
     ]
    },
    {
     "id": "2026-10-10T13-18-13",
     "set": "holdout-a",
     "setLabel": "Held-out A",
     "agent": "Claude Code",
     "model": "Sonnet",
     "note": null,
     "rerun": false,
     "tasks": 24,
     "repeats": 2,
     "raw": {
      "runs": 48,
      "resolve": 0.917,
      "turns": 10.4,
      "cost": 0.0741,
      "seconds": 25.3,
      "outOfScope": 0.08,
      "errors": 0
     },
     "v8": {
      "runs": 48,
      "resolve": 0.917,
      "turns": 8.7,
      "cost": 0.0866,
      "seconds": 29.1,
      "outOfScope": 1.75,
      "errors": 0
     },
     "tally": {
      "win": 2,
      "tie": 20,
      "loss": 2
     },
     "categories": {
      "bugfix": {
       "tasks": 8,
       "raw": 0.875,
       "v8": 0.875
      },
      "feature": {
       "tasks": 11,
       "raw": 1,
       "v8": 0.909
      },
      "refactor": {
       "tasks": 3,
       "raw": 0.667,
       "v8": 1
      },
      "scope": {
       "tasks": 2,
       "raw": 1,
       "v8": 1
      }
     },
     "results": [
      {
       "id": "ho-a-auth-lockout",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-a-basket-totals",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "ho-a-invoice-due-date",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-a-pulse-sessions",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-a-rota-overnight-crash",
       "category": "bugfix",
       "language": "js",
       "raw": 0,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "ho-a-stock-release-tr",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-b-helpdesk-sla-due",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-b-leave-csv-export",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-b-library-renew-tr",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-b-newsletter-batch-limit",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-b-shipment-status-api",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-b-telemetry-zero-division",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-c-ci-rename-workers",
       "category": "refactor",
       "language": "js",
       "raw": 0,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "ho-c-docstore-list-order",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-c-podcast-retry-helper",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-c-recipe-volume-units",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "ho-c-timetable-room-labels",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-c-tr-fitness-durations",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-d-budget-idempotency",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-d-clinic-no-show",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-d-ticket-transfer-spec",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-d-tm-fuzzy-deprecated",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-d-tr-leaderboard-weekly",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "ho-d-warehouse-picklist-perf",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      }
     ]
    }
   ]
  },
  {
   "id": "c2026.10.16",
   "date": "2026-10-10",
   "cli": "0.5.0",
   "status": "measured",
   "title": "Specs pass through untouched",
   "summary": "A request that is already a precise spec goes to the agent verbatim. A fix brief never also forbids changes, and a cause the user guessed is marked as a guess.",
   "changes": [
    "Precise specs attached as written",
    "No 'do not modify files' in a fix",
    "The user's diagnosis is a lead, not a fact",
    "No fences around files the user never named"
   ],
   "studies": [
    {
     "id": "2026-10-10T15-47-52",
     "set": "holdout-b",
     "setLabel": "Held-out B",
     "agent": "Claude Code",
     "model": "Sonnet",
     "note": null,
     "rerun": false,
     "tasks": 24,
     "repeats": 2,
     "raw": {
      "runs": 48,
      "resolve": 0.979,
      "turns": 11.5,
      "cost": 0.0914,
      "seconds": 28.2,
      "outOfScope": 0,
      "errors": 0
     },
     "v8": {
      "runs": 48,
      "resolve": 0.917,
      "turns": 9,
      "cost": 0.096,
      "seconds": 28.8,
      "outOfScope": 0,
      "errors": 0
     },
     "tally": {
      "win": 1,
      "tie": 21,
      "loss": 2
     },
     "categories": {
      "feature": {
       "tasks": 9,
       "raw": 1,
       "v8": 0.889
      },
      "bugfix": {
       "tasks": 10,
       "raw": 0.95,
       "v8": 1
      },
      "refactor": {
       "tasks": 3,
       "raw": 1,
       "v8": 1
      },
      "scope": {
       "tasks": 2,
       "raw": 1,
       "v8": 0.5
      }
     },
     "results": [
      {
       "id": "hb-1-garage-overstays",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-hotel-feed-crash",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-quiz-multiselect-tr",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-refill-controlled-early",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-registry-deprecate",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "hb-1-smarthome-quiet-hours",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-desk-overlap-tr",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-flags-rollout",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-foodbank-export",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-league-layering",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-podcast-tidy",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-vet-reminders",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-bikeshare-nearest-options",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-irrigation-imperial-forecast",
       "category": "bugfix",
       "language": "py",
       "raw": 0.5,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "hb-3-legal-court-deadlines",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-library-holds-tr",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-museum-timeslot-pagination",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-recruiting-offer-webhook",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "hb-4-js-survey-rating-type",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-js-tr-table-best-fit",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-js-waitlist-skips",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-firmware-waves-spec",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-loyalty-field-rename",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-permit-deadline-shown",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      }
     ]
    }
   ],
   "metrics": []
  },
  {
   "id": "c2026.10.17",
   "date": "2026-10-10",
   "cli": "0.5.0",
   "status": "measured",
   "title": "Needs are requests",
   "summary": "'We need to ping HR when…' is a change request, not a question. 'Ask before' keeps risky actions only.",
   "changes": [
    "Stated needs compile as change requests",
    "'Ask before' limited to data, schema, dependencies, deploys"
   ],
   "studies": [
    {
     "id": "2026-10-10T19-50-00",
     "set": "holdout-b",
     "setLabel": "Held-out B",
     "agent": "Claude Code",
     "model": "Sonnet",
     "note": "v8 condition rerun on the same raw runs",
     "rerun": true,
     "tasks": 24,
     "repeats": 2,
     "raw": {
      "runs": 48,
      "resolve": 0.979,
      "turns": 11.5,
      "cost": 0.0914,
      "seconds": 28.2,
      "outOfScope": 0,
      "errors": 0
     },
     "v8": {
      "runs": 48,
      "resolve": 0.958,
      "turns": 9.1,
      "cost": 0.0971,
      "seconds": 32.2,
      "outOfScope": 0,
      "errors": 0
     },
     "tally": {
      "win": 1,
      "tie": 22,
      "loss": 1
     },
     "categories": {
      "feature": {
       "tasks": 9,
       "raw": 1,
       "v8": 1
      },
      "bugfix": {
       "tasks": 10,
       "raw": 0.95,
       "v8": 1
      },
      "refactor": {
       "tasks": 3,
       "raw": 1,
       "v8": 0.667
      },
      "scope": {
       "tasks": 2,
       "raw": 1,
       "v8": 1
      }
     },
     "results": [
      {
       "id": "hb-1-garage-overstays",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-hotel-feed-crash",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-quiz-multiselect-tr",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-refill-controlled-early",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-registry-deprecate",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-smarthome-quiet-hours",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-desk-overlap-tr",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-flags-rollout",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-foodbank-export",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-league-layering",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-podcast-tidy",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-vet-reminders",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-bikeshare-nearest-options",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "hb-3-irrigation-imperial-forecast",
       "category": "bugfix",
       "language": "py",
       "raw": 0.5,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "hb-3-legal-court-deadlines",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-library-holds-tr",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-museum-timeslot-pagination",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-recruiting-offer-webhook",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-js-survey-rating-type",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-js-tr-table-best-fit",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-js-waitlist-skips",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-firmware-waves-spec",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-loyalty-field-rename",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-permit-deadline-shown",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      }
     ]
    }
   ],
   "metrics": []
  },
  {
   "id": "c2026.10.18",
   "date": "2026-10-10",
   "cli": "0.5.0",
   "status": "shipped",
   "title": "No invented scopes",
   "summary": "'Limit changes to the listed files' and every other file scope the user never set is dropped.",
   "changes": [
    "Invented file scopes dropped in every wording"
   ],
   "studies": [
    {
     "id": "2026-10-10T17-17-29",
     "set": "holdout-b",
     "setLabel": "Held-out B",
     "agent": "Claude Code",
     "model": "Haiku",
     "note": null,
     "rerun": false,
     "tasks": 24,
     "repeats": 2,
     "raw": {
      "runs": 48,
      "resolve": 1,
      "turns": 13.4,
      "cost": 0.0091,
      "seconds": 40.1,
      "outOfScope": 0,
      "errors": 0
     },
     "v8": {
      "runs": 48,
      "resolve": 0.896,
      "turns": 14,
      "cost": 0.01,
      "seconds": 44.4,
      "outOfScope": 0,
      "errors": 0
     },
     "tally": {
      "win": 0,
      "tie": 21,
      "loss": 3
     },
     "categories": {
      "feature": {
       "tasks": 9,
       "raw": 1,
       "v8": 0.889
      },
      "bugfix": {
       "tasks": 10,
       "raw": 1,
       "v8": 1
      },
      "refactor": {
       "tasks": 3,
       "raw": 1,
       "v8": 0.5
      },
      "scope": {
       "tasks": 2,
       "raw": 1,
       "v8": 1
      }
     },
     "results": [
      {
       "id": "hb-1-garage-overstays",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-hotel-feed-crash",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-quiz-multiselect-tr",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-refill-controlled-early",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-registry-deprecate",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "hb-1-smarthome-quiet-hours",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-desk-overlap-tr",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-flags-rollout",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-foodbank-export",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-league-layering",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 0.5,
       "outcome": "loss"
      },
      {
       "id": "hb-2-podcast-tidy",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-vet-reminders",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-bikeshare-nearest-options",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "hb-3-irrigation-imperial-forecast",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-legal-court-deadlines",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-library-holds-tr",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-museum-timeslot-pagination",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-recruiting-offer-webhook",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-js-survey-rating-type",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-js-tr-table-best-fit",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-js-waitlist-skips",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-firmware-waves-spec",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-loyalty-field-rename",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-permit-deadline-shown",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      }
     ]
    }
   ],
   "metrics": []
  },
  {
   "id": "c2026.10.19",
   "date": "2026-10-11",
   "cli": "0.5.0",
   "status": "measured",
   "title": "Your words travel with the brief",
   "summary": "Every coding brief carries the request exactly as written, and the request wins wherever the brief reads differently. The losses we kept seeing were paraphrase drift, not missing context.",
   "changes": [
    "Request attached verbatim to every coding brief",
    "Where they differ, the request wins"
   ],
   "studies": [
    {
     "id": "2026-10-11T09-00-00",
     "set": "holdout-b",
     "setLabel": "Held-out B",
     "agent": "Claude Code",
     "model": "Haiku",
     "note": "v8 condition rerun on the same raw runs",
     "rerun": true,
     "tasks": 24,
     "repeats": 2,
     "raw": {
      "runs": 48,
      "resolve": 1,
      "turns": 13.4,
      "cost": 0.0091,
      "seconds": 40.1,
      "outOfScope": 0,
      "errors": 0
     },
     "v8": {
      "runs": 48,
      "resolve": 1,
      "turns": 13.8,
      "cost": 0.0099,
      "seconds": 43.7,
      "outOfScope": 0.04,
      "errors": 0
     },
     "tally": {
      "win": 0,
      "tie": 24,
      "loss": 0
     },
     "categories": {
      "feature": {
       "tasks": 9,
       "raw": 1,
       "v8": 1
      },
      "bugfix": {
       "tasks": 10,
       "raw": 1,
       "v8": 1
      },
      "refactor": {
       "tasks": 3,
       "raw": 1,
       "v8": 1
      },
      "scope": {
       "tasks": 2,
       "raw": 1,
       "v8": 1
      }
     },
     "results": [
      {
       "id": "hb-1-garage-overstays",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-hotel-feed-crash",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-quiz-multiselect-tr",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-refill-controlled-early",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-registry-deprecate",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-1-smarthome-quiet-hours",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-desk-overlap-tr",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-flags-rollout",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-foodbank-export",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-league-layering",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-podcast-tidy",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-2-vet-reminders",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-bikeshare-nearest-options",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-irrigation-imperial-forecast",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-legal-court-deadlines",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-library-holds-tr",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-museum-timeslot-pagination",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-3-recruiting-offer-webhook",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-js-survey-rating-type",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-js-tr-table-best-fit",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-js-waitlist-skips",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-firmware-waves-spec",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-loyalty-field-rename",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hb-4-py-permit-deadline-shown",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      }
     ]
    },
    {
     "id": "2026-10-10T23-28-04",
     "set": "holdout-c",
     "setLabel": "Held-out C",
     "agent": "Claude Code",
     "model": "Sonnet",
     "note": null,
     "rerun": false,
     "tasks": 24,
     "repeats": 2,
     "raw": {
      "runs": 48,
      "resolve": 0.896,
      "turns": 13.6,
      "cost": 0.0954,
      "seconds": 34.8,
      "outOfScope": 0,
      "errors": 0
     },
     "v8": {
      "runs": 48,
      "resolve": 0.917,
      "turns": 11.3,
      "cost": 0.1097,
      "seconds": 42.2,
      "outOfScope": 1.46,
      "errors": 0
     },
     "tally": {
      "win": 1,
      "tie": 23,
      "loss": 0
     },
     "categories": {
      "refactor": {
       "tasks": 2,
       "raw": 1,
       "v8": 1
      },
      "bugfix": {
       "tasks": 11,
       "raw": 1,
       "v8": 1
      },
      "feature": {
       "tasks": 8,
       "raw": 0.813,
       "v8": 0.875
      },
      "scope": {
       "tasks": 3,
       "raw": 0.667,
       "v8": 0.667
      }
     },
     "results": [
      {
       "id": "hc-1-claims-status-transitions",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-1-dispatch-new-drivers-500",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-1-energy-v1-totals",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-1-fleet-next-service",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-1-newsroom-author-filter",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-1-ticketing-checkin-names-tr",
       "category": "scope",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-2-customs-cn23-weights",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-2-forum-report-autohide-tr",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-2-grading-excused-quiz",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-2-itinerary-overlap-warning",
       "category": "feature",
       "language": "js",
       "raw": 0.5,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "hc-2-parking-spots-stuck-taken",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-2-ward-isolation-rule",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-3-js-collar-silence",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-3-js-lookaway-detector",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-3-js-route-sheet-flaky",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-3-py-monthly-lapsed",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-3-py-royalty-suspense",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-3-py-spc-alarm-tr",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-4-coop-priority-tidy",
       "category": "scope",
       "language": "js",
       "raw": 0,
       "v8": 0,
       "outcome": "tie"
      },
      {
       "id": "hc-4-dental-repeat-lines",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-4-labels-postcode-batch",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-4-matchmaking-wait-tr",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-4-quake-zone-threshold",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-4-translation-review-skip",
       "category": "feature",
       "language": "py",
       "raw": 0,
       "v8": 0,
       "outcome": "tie"
      }
     ]
    },
    {
     "id": "2026-10-10T23-28-10",
     "set": "holdout-c",
     "setLabel": "Held-out C",
     "agent": "Claude Code",
     "model": "Haiku",
     "note": null,
     "rerun": false,
     "tasks": 24,
     "repeats": 2,
     "raw": {
      "runs": 48,
      "resolve": 0.896,
      "turns": 15.2,
      "cost": 0.0102,
      "seconds": 47.8,
      "outOfScope": 0.13,
      "errors": 0
     },
     "v8": {
      "runs": 48,
      "resolve": 0.875,
      "turns": 15.5,
      "cost": 0.0126,
      "seconds": 83.4,
      "outOfScope": 1.94,
      "errors": 0
     },
     "tally": {
      "win": 2,
      "tie": 20,
      "loss": 2
     },
     "categories": {
      "refactor": {
       "tasks": 2,
       "raw": 1,
       "v8": 0.75
      },
      "bugfix": {
       "tasks": 11,
       "raw": 0.955,
       "v8": 0.909
      },
      "feature": {
       "tasks": 8,
       "raw": 0.875,
       "v8": 0.875
      },
      "scope": {
       "tasks": 3,
       "raw": 0.667,
       "v8": 0.833
      }
     },
     "results": [
      {
       "id": "hc-1-claims-status-transitions",
       "category": "refactor",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-1-dispatch-new-drivers-500",
       "category": "bugfix",
       "language": "js",
       "raw": 0.5,
       "v8": 1,
       "outcome": "win"
      },
      {
       "id": "hc-1-energy-v1-totals",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-1-fleet-next-service",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-1-newsroom-author-filter",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-1-ticketing-checkin-names-tr",
       "category": "scope",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-2-customs-cn23-weights",
       "category": "scope",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-2-forum-report-autohide-tr",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-2-grading-excused-quiz",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-2-itinerary-overlap-warning",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-2-parking-spots-stuck-taken",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-2-ward-isolation-rule",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-3-js-collar-silence",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 0,
       "outcome": "loss"
      },
      {
       "id": "hc-3-js-lookaway-detector",
       "category": "refactor",
       "language": "js",
       "raw": 1,
       "v8": 0.5,
       "outcome": "loss"
      },
      {
       "id": "hc-3-js-route-sheet-flaky",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-3-py-monthly-lapsed",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-3-py-royalty-suspense",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-3-py-spc-alarm-tr",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-4-coop-priority-tidy",
       "category": "scope",
       "language": "js",
       "raw": 0,
       "v8": 0.5,
       "outcome": "win"
      },
      {
       "id": "hc-4-dental-repeat-lines",
       "category": "feature",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-4-labels-postcode-batch",
       "category": "bugfix",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-4-matchmaking-wait-tr",
       "category": "feature",
       "language": "js",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-4-quake-zone-threshold",
       "category": "bugfix",
       "language": "py",
       "raw": 1,
       "v8": 1,
       "outcome": "tie"
      },
      {
       "id": "hc-4-translation-review-skip",
       "category": "feature",
       "language": "py",
       "raw": 0,
       "v8": 0,
       "outcome": "tie"
      }
     ]
    }
   ],
   "metrics": []
  },
  {
   "id": "c2026.10.20",
   "date": "2026-10-11",
   "cli": "0.5.0",
   "status": "shipped",
   "title": "Reports are not questions",
   "summary": "A request that describes a behaviour ('the gap report says 0.1 h, it was dark for three days, something's off') is a fix, not a read-only investigation. In held-out set C that one misreading cost Haiku a task raw agents solved twice.",
   "changes": [
    "Described behaviour compiles as a fix",
    "Read-only only when the request asks to read, or is a short topic",
    "\"something's off\" counts as a defect report"
   ],
   "studies": [],
   "metrics": []
  }
 ],
 "totals": {
  "versions": 9,
  "studies": 9,
  "agentRuns": 876,
  "tasks": 122
 }
}