{
  "n": 200,
  "images": 73,
  "accuracy_all": 0.81,
  "accuracy": {
    "noul": 0.8202247191011236,
    "choice": 0.8018018018018018
  },
  "nll": 0.4400090990936913,
  "brier": 0.25543111423553433,
  "ece": 0.06582706868648526,
  "ece_bins": 15,
  "reliability": [
    {
      "bin": [
        0.2,
        0.3
      ],
      "n": 1,
      "confidence": 0.28821372985839844,
      "accuracy": 1.0
    },
    {
      "bin": [
        0.3,
        0.4
      ],
      "n": 5,
      "confidence": 0.3615827262401581,
      "accuracy": 0.4
    },
    {
      "bin": [
        0.4,
        0.5
      ],
      "n": 7,
      "confidence": 0.4423674430165972,
      "accuracy": 0.42857142857142855
    },
    {
      "bin": [
        0.5,
        0.6
      ],
      "n": 33,
      "confidence": 0.5360189441478613,
      "accuracy": 0.6363636363636364
    },
    {
      "bin": [
        0.6,
        0.7
      ],
      "n": 26,
      "confidence": 0.6526116694395359,
      "accuracy": 0.6923076923076923
    },
    {
      "bin": [
        0.7,
        0.8
      ],
      "n": 15,
      "confidence": 0.7433490335941315,
      "accuracy": 0.8
    },
    {
      "bin": [
        0.8,
        0.9
      ],
      "n": 38,
      "confidence": 0.8569445978654059,
      "accuracy": 0.8421052631578947
    },
    {
      "bin": [
        0.9,
        1.0
      ],
      "n": 75,
      "confidence": 0.953959968884786,
      "accuracy": 0.9733333333333334
    }
  ],
  "thresholds": {
    "0.6": {
      "coverage": 0.77,
      "accuracy": 0.8766233766233766
    },
    "0.7": {
      "coverage": 0.64,
      "accuracy": 0.9140625
    },
    "0.8": {
      "coverage": 0.565,
      "accuracy": 0.9292035398230089
    },
    "0.9": {
      "coverage": 0.375,
      "accuracy": 0.9733333333333334
    }
  },
  "by_source": {
    "ai2d": {
      "n": 2,
      "images": 1,
      "accuracy": 0.5,
      "nll": 1.2898404388867466,
      "ece": 0.3732716739177704
    },
    "aokvqa": {
      "n": 4,
      "images": 2,
      "accuracy": 0.75,
      "nll": 0.611274152606737,
      "ece": 0.2411895990371704
    },
    "coco-exist": {
      "n": 39,
      "images": 25,
      "accuracy": 0.9230769230769231,
      "nll": 0.23563569869662415,
      "ece": 0.0540550612868407
    },
    "coco-position": {
      "n": 26,
      "images": 18,
      "accuracy": 0.9230769230769231,
      "nll": 0.20050254680859508,
      "ece": 0.12922574006594145
    },
    "coco-relation": {
      "n": 10,
      "images": 5,
      "accuracy": 0.8,
      "nll": 0.4535431995357432,
      "ece": 0.29919506013393404
    },
    "coco-size": {
      "n": 2,
      "images": 2,
      "accuracy": 1.0,
      "nll": 0.3333616534201184,
      "ece": 0.2508668899536133
    },
    "gqa-v2": {
      "n": 25,
      "images": 7,
      "accuracy": 0.68,
      "nll": 0.5026796189257893,
      "ece": 0.21928748846054075
    },
    "tqa": {
      "n": 10,
      "images": 2,
      "accuracy": 0.5,
      "nll": 1.2075195747510135,
      "ece": 0.35332188010215765
    },
    "vqav2-choice": {
      "n": 35,
      "images": 21,
      "accuracy": 0.8857142857142857,
      "nll": 0.3503283097869317,
      "ece": 0.15751930900982447
    },
    "vqav2-yesno": {
      "n": 38,
      "images": 24,
      "accuracy": 0.7368421052631579,
      "nll": 0.5690740238980908,
      "ece": 0.11526736793549436
    },
    "vsr": {
      "n": 9,
      "images": 4,
      "accuracy": 0.7777777777777778,
      "nll": 0.5381709545359337,
      "ece": 0.2373169246647093
    }
  },
  "by_kind": {
    "ai2d": {
      "n": 2,
      "images": 1,
      "accuracy": 0.5,
      "nll": 1.2898404388867466,
      "ece": 0.3732716739177704
    },
    "animal": {
      "n": 2,
      "images": 2,
      "accuracy": 1.0,
      "nll": 0.08400760063551806,
      "ece": 0.07780307531356812
    },
    "aokvqa": {
      "n": 4,
      "images": 2,
      "accuracy": 0.75,
      "nll": 0.611274152606737,
      "ece": 0.2411895990371704
    },
    "color": {
      "n": 14,
      "images": 11,
      "accuracy": 0.8571428571428571,
      "nll": 0.3618174527077554,
      "ece": 0.09759957662650516
    },
    "count": {
      "n": 10,
      "images": 7,
      "accuracy": 0.8,
      "nll": 0.6097179990869517,
      "ece": 0.3152927696704865
    },
    "exist": {
      "n": 39,
      "images": 25,
      "accuracy": 0.9230769230769231,
      "nll": 0.23563569869662415,
      "ece": 0.0540550612868407
    },
    "food": {
      "n": 1,
      "images": 1,
      "accuracy": 1.0,
      "nll": 0.0042057737329474765,
      "ece": 0.00419694185256958
    },
    "gqa": {
      "n": 11,
      "images": 6,
      "accuracy": 0.7272727272727273,
      "nll": 0.5031112251493984,
      "ece": 0.2824250134554776
    },
    "material": {
      "n": 2,
      "images": 2,
      "accuracy": 1.0,
      "nll": 0.22872908175288753,
      "ece": 0.19662755727767944
    },
    "meat": {
      "n": 1,
      "images": 1,
      "accuracy": 1.0,
      "nll": 0.1798803908020786,
      "ece": 0.16462987661361694
    },
    "position": {
      "n": 26,
      "images": 18,
      "accuracy": 0.9230769230769231,
      "nll": 0.20050254680859508,
      "ece": 0.12922574006594145
    },
    "relation": {
      "n": 10,
      "images": 5,
      "accuracy": 0.8,
      "nll": 0.4535431995357432,
      "ece": 0.29919506013393404
    },
    "room": {
      "n": 1,
      "images": 1,
      "accuracy": 1.0,
      "nll": 0.005263986423792628,
      "ece": 0.005250155925750732
    },
    "size": {
      "n": 2,
      "images": 2,
      "accuracy": 1.0,
      "nll": 0.3333616534201184,
      "ece": 0.2508668899536133
    },
    "spatial": {
      "n": 14,
      "images": 5,
      "accuracy": 0.6428571428571429,
      "nll": 0.5023404997500963,
      "ece": 0.1696794331073761
    },
    "tqa": {
      "n": 10,
      "images": 2,
      "accuracy": 0.5,
      "nll": 1.2075195747510135,
      "ece": 0.35332188010215765
    },
    "vegetable": {
      "n": 4,
      "images": 4,
      "accuracy": 1.0,
      "nll": 0.07101074950722219,
      "ece": 0.06772895157337189
    },
    "vsr": {
      "n": 9,
      "images": 4,
      "accuracy": 0.7777777777777778,
      "nll": 0.5381709545359337,
      "ece": 0.2373169246647093
    },
    "yesno": {
      "n": 38,
      "images": 24,
      "accuracy": 0.7368421052631579,
      "nll": 0.5690740238980908,
      "ece": 0.11526736793549436
    }
  },
  "by_axis": {
    "-": {
      "n": 162,
      "images": 67,
      "accuracy": 0.7901234567901234,
      "nll": 0.47554891340501837,
      "ece": 0.05595421993438109
    },
    "lr": {
      "n": 27,
      "images": 18,
      "accuracy": 0.8888888888888888,
      "nll": 0.3277449296076739,
      "ece": 0.19794687407988087
    },
    "tb": {
      "n": 11,
      "images": 11,
      "accuracy": 0.9090909090909091,
      "nll": 0.1921620679743721,
      "ece": 0.09886410019614479
    }
  },
  "by_options": {
    "2": {
      "n": 157,
      "images": 64,
      "accuracy": 0.8280254777070064,
      "nll": 0.38058168910086837,
      "ece": 0.06867314277181197
    },
    "3": {
      "n": 10,
      "images": 9,
      "accuracy": 0.9,
      "nll": 0.4393064845098579,
      "ece": 0.22365574836730956
    },
    "4": {
      "n": 31,
      "images": 18,
      "accuracy": 0.6774193548387096,
      "nll": 0.7616504939115978,
      "ece": 0.10811415699220474
    },
    "5": {
      "n": 2,
      "images": 2,
      "accuracy": 1.0,
      "nll": 0.12313223677190567,
      "ece": 0.1151973307132721
    }
  },
  "noul_confusion": {
    "tp": 42,
    "fp": 10,
    "fn": 6,
    "tn": 31,
    "yes_ratio": 0.5842696629213483
  },
  "request_latency_ms": {
    "p50": 174.91745902225375,
    "p95": 189.9587146937847,
    "requests": 200,
    "warmup_excluded": 3
  },
  "questions_per_request": {
    "mean": 1.0,
    "p50": 1.0,
    "max": 1
  },
  "ms_per_question": {
    "p50": 174.91745902225375,
    "p95": 189.9587146937847,
    "overall": 172.57170106749982
  },
  "latency_ms": {
    "p50": 174.91745902225375,
    "p95": 189.9587146937847,
    "requests": 200,
    "warmup_excluded": 3
  },
  "control": "none",
  "model": "devision-0.1",
  "temperature": {
    "choice": 1.5922062397003174,
    "noul": 1.825764536857605,
    "by_options": {
      "choice:11+": 1.201171636581421,
      "choice:2": 1.9487814903259277,
      "choice:3-5": 1.7575252056121826,
      "noul:2": 1.825764536857605
    }
  },
  "grouping": {
    "max_questions_per_request": 1,
    "by": "none (one question per request)"
  },
  "environment": {
    "threads": 4,
    "machine": "arm64",
    "system": "Darwin"
  },
  "run": {
    "checkpoint": "runs/r15-vision/stage2",
    "data": "data/r12/cpu_latency.jsonl",
    "data_sha256": "352f0b4676aeb7389b45a93086169096d6b8e3962568543cfa373877c3e84020",
    "role": "unspecified",
    "device": "cpu",
    "temperature": "fitted",
    "control": "none",
    "seed": 0,
    "code": "6feb0b3+dirty"
  }
}