| { |
| "n": 200, |
| "images": 73, |
| "accuracy_all": 0.81, |
| "accuracy": { |
| "noul": 0.8202247191011236, |
| "choice": 0.8018018018018018 |
| }, |
| "nll": 0.4400090990936913, |
| "brier": 0.25543111423553433, |
| "ece": 0.06582706868648526, |
| "ece_bins": 15, |
| "reliability": [ |
| { |
| "bin": [ |
| 0.2, |
| 0.3 |
| ], |
| "n": 1, |
| "confidence": 0.28821372985839844, |
| "accuracy": 1.0 |
| }, |
| { |
| "bin": [ |
| 0.3, |
| 0.4 |
| ], |
| "n": 5, |
| "confidence": 0.3615827262401581, |
| "accuracy": 0.4 |
| }, |
| { |
| "bin": [ |
| 0.4, |
| 0.5 |
| ], |
| "n": 7, |
| "confidence": 0.4423674430165972, |
| "accuracy": 0.42857142857142855 |
| }, |
| { |
| "bin": [ |
| 0.5, |
| 0.6 |
| ], |
| "n": 33, |
| "confidence": 0.5360189441478613, |
| "accuracy": 0.6363636363636364 |
| }, |
| { |
| "bin": [ |
| 0.6, |
| 0.7 |
| ], |
| "n": 26, |
| "confidence": 0.6526116694395359, |
| "accuracy": 0.6923076923076923 |
| }, |
| { |
| "bin": [ |
| 0.7, |
| 0.8 |
| ], |
| "n": 15, |
| "confidence": 0.7433490335941315, |
| "accuracy": 0.8 |
| }, |
| { |
| "bin": [ |
| 0.8, |
| 0.9 |
| ], |
| "n": 38, |
| "confidence": 0.8569445978654059, |
| "accuracy": 0.8421052631578947 |
| }, |
| { |
| "bin": [ |
| 0.9, |
| 1.0 |
| ], |
| "n": 75, |
| "confidence": 0.953959968884786, |
| "accuracy": 0.9733333333333334 |
| } |
| ], |
| "thresholds": { |
| "0.6": { |
| "coverage": 0.77, |
| "accuracy": 0.8766233766233766 |
| }, |
| "0.7": { |
| "coverage": 0.64, |
| "accuracy": 0.9140625 |
| }, |
| "0.8": { |
| "coverage": 0.565, |
| "accuracy": 0.9292035398230089 |
| }, |
| "0.9": { |
| "coverage": 0.375, |
| "accuracy": 0.9733333333333334 |
| } |
| }, |
| "by_source": { |
| "ai2d": { |
| "n": 2, |
| "images": 1, |
| "accuracy": 0.5, |
| "nll": 1.2898404388867466, |
| "ece": 0.3732716739177704 |
| }, |
| "aokvqa": { |
| "n": 4, |
| "images": 2, |
| "accuracy": 0.75, |
| "nll": 0.611274152606737, |
| "ece": 0.2411895990371704 |
| }, |
| "coco-exist": { |
| "n": 39, |
| "images": 25, |
| "accuracy": 0.9230769230769231, |
| "nll": 0.23563569869662415, |
| "ece": 0.0540550612868407 |
| }, |
| "coco-position": { |
| "n": 26, |
| "images": 18, |
| "accuracy": 0.9230769230769231, |
| "nll": 0.20050254680859508, |
| "ece": 0.12922574006594145 |
| }, |
| "coco-relation": { |
| "n": 10, |
| "images": 5, |
| "accuracy": 0.8, |
| "nll": 0.4535431995357432, |
| "ece": 0.29919506013393404 |
| }, |
| "coco-size": { |
| "n": 2, |
| "images": 2, |
| "accuracy": 1.0, |
| "nll": 0.3333616534201184, |
| "ece": 0.2508668899536133 |
| }, |
| "gqa-v2": { |
| "n": 25, |
| "images": 7, |
| "accuracy": 0.68, |
| "nll": 0.5026796189257893, |
| "ece": 0.21928748846054075 |
| }, |
| "tqa": { |
| "n": 10, |
| "images": 2, |
| "accuracy": 0.5, |
| "nll": 1.2075195747510135, |
| "ece": 0.35332188010215765 |
| }, |
| "vqav2-choice": { |
| "n": 35, |
| "images": 21, |
| "accuracy": 0.8857142857142857, |
| "nll": 0.3503283097869317, |
| "ece": 0.15751930900982447 |
| }, |
| "vqav2-yesno": { |
| "n": 38, |
| "images": 24, |
| "accuracy": 0.7368421052631579, |
| "nll": 0.5690740238980908, |
| "ece": 0.11526736793549436 |
| }, |
| "vsr": { |
| "n": 9, |
| "images": 4, |
| "accuracy": 0.7777777777777778, |
| "nll": 0.5381709545359337, |
| "ece": 0.2373169246647093 |
| } |
| }, |
| "by_kind": { |
| "ai2d": { |
| "n": 2, |
| "images": 1, |
| "accuracy": 0.5, |
| "nll": 1.2898404388867466, |
| "ece": 0.3732716739177704 |
| }, |
| "animal": { |
| "n": 2, |
| "images": 2, |
| "accuracy": 1.0, |
| "nll": 0.08400760063551806, |
| "ece": 0.07780307531356812 |
| }, |
| "aokvqa": { |
| "n": 4, |
| "images": 2, |
| "accuracy": 0.75, |
| "nll": 0.611274152606737, |
| "ece": 0.2411895990371704 |
| }, |
| "color": { |
| "n": 14, |
| "images": 11, |
| "accuracy": 0.8571428571428571, |
| "nll": 0.3618174527077554, |
| "ece": 0.09759957662650516 |
| }, |
| "count": { |
| "n": 10, |
| "images": 7, |
| "accuracy": 0.8, |
| "nll": 0.6097179990869517, |
| "ece": 0.3152927696704865 |
| }, |
| "exist": { |
| "n": 39, |
| "images": 25, |
| "accuracy": 0.9230769230769231, |
| "nll": 0.23563569869662415, |
| "ece": 0.0540550612868407 |
| }, |
| "food": { |
| "n": 1, |
| "images": 1, |
| "accuracy": 1.0, |
| "nll": 0.0042057737329474765, |
| "ece": 0.00419694185256958 |
| }, |
| "gqa": { |
| "n": 11, |
| "images": 6, |
| "accuracy": 0.7272727272727273, |
| "nll": 0.5031112251493984, |
| "ece": 0.2824250134554776 |
| }, |
| "material": { |
| "n": 2, |
| "images": 2, |
| "accuracy": 1.0, |
| "nll": 0.22872908175288753, |
| "ece": 0.19662755727767944 |
| }, |
| "meat": { |
| "n": 1, |
| "images": 1, |
| "accuracy": 1.0, |
| "nll": 0.1798803908020786, |
| "ece": 0.16462987661361694 |
| }, |
| "position": { |
| "n": 26, |
| "images": 18, |
| "accuracy": 0.9230769230769231, |
| "nll": 0.20050254680859508, |
| "ece": 0.12922574006594145 |
| }, |
| "relation": { |
| "n": 10, |
| "images": 5, |
| "accuracy": 0.8, |
| "nll": 0.4535431995357432, |
| "ece": 0.29919506013393404 |
| }, |
| "room": { |
| "n": 1, |
| "images": 1, |
| "accuracy": 1.0, |
| "nll": 0.005263986423792628, |
| "ece": 0.005250155925750732 |
| }, |
| "size": { |
| "n": 2, |
| "images": 2, |
| "accuracy": 1.0, |
| "nll": 0.3333616534201184, |
| "ece": 0.2508668899536133 |
| }, |
| "spatial": { |
| "n": 14, |
| "images": 5, |
| "accuracy": 0.6428571428571429, |
| "nll": 0.5023404997500963, |
| "ece": 0.1696794331073761 |
| }, |
| "tqa": { |
| "n": 10, |
| "images": 2, |
| "accuracy": 0.5, |
| "nll": 1.2075195747510135, |
| "ece": 0.35332188010215765 |
| }, |
| "vegetable": { |
| "n": 4, |
| "images": 4, |
| "accuracy": 1.0, |
| "nll": 0.07101074950722219, |
| "ece": 0.06772895157337189 |
| }, |
| "vsr": { |
| "n": 9, |
| "images": 4, |
| "accuracy": 0.7777777777777778, |
| "nll": 0.5381709545359337, |
| "ece": 0.2373169246647093 |
| }, |
| "yesno": { |
| "n": 38, |
| "images": 24, |
| "accuracy": 0.7368421052631579, |
| "nll": 0.5690740238980908, |
| "ece": 0.11526736793549436 |
| } |
| }, |
| "by_axis": { |
| "-": { |
| "n": 162, |
| "images": 67, |
| "accuracy": 0.7901234567901234, |
| "nll": 0.47554891340501837, |
| "ece": 0.05595421993438109 |
| }, |
| "lr": { |
| "n": 27, |
| "images": 18, |
| "accuracy": 0.8888888888888888, |
| "nll": 0.3277449296076739, |
| "ece": 0.19794687407988087 |
| }, |
| "tb": { |
| "n": 11, |
| "images": 11, |
| "accuracy": 0.9090909090909091, |
| "nll": 0.1921620679743721, |
| "ece": 0.09886410019614479 |
| } |
| }, |
| "by_options": { |
| "2": { |
| "n": 157, |
| "images": 64, |
| "accuracy": 0.8280254777070064, |
| "nll": 0.38058168910086837, |
| "ece": 0.06867314277181197 |
| }, |
| "3": { |
| "n": 10, |
| "images": 9, |
| "accuracy": 0.9, |
| "nll": 0.4393064845098579, |
| "ece": 0.22365574836730956 |
| }, |
| "4": { |
| "n": 31, |
| "images": 18, |
| "accuracy": 0.6774193548387096, |
| "nll": 0.7616504939115978, |
| "ece": 0.10811415699220474 |
| }, |
| "5": { |
| "n": 2, |
| "images": 2, |
| "accuracy": 1.0, |
| "nll": 0.12313223677190567, |
| "ece": 0.1151973307132721 |
| } |
| }, |
| "noul_confusion": { |
| "tp": 42, |
| "fp": 10, |
| "fn": 6, |
| "tn": 31, |
| "yes_ratio": 0.5842696629213483 |
| }, |
| "request_latency_ms": { |
| "p50": 174.91745902225375, |
| "p95": 189.9587146937847, |
| "requests": 200, |
| "warmup_excluded": 3 |
| }, |
| "questions_per_request": { |
| "mean": 1.0, |
| "p50": 1.0, |
| "max": 1 |
| }, |
| "ms_per_question": { |
| "p50": 174.91745902225375, |
| "p95": 189.9587146937847, |
| "overall": 172.57170106749982 |
| }, |
| "latency_ms": { |
| "p50": 174.91745902225375, |
| "p95": 189.9587146937847, |
| "requests": 200, |
| "warmup_excluded": 3 |
| }, |
| "control": "none", |
| "model": "devision-0.1", |
| "temperature": { |
| "choice": 1.5922062397003174, |
| "noul": 1.825764536857605, |
| "by_options": { |
| "choice:11+": 1.201171636581421, |
| "choice:2": 1.9487814903259277, |
| "choice:3-5": 1.7575252056121826, |
| "noul:2": 1.825764536857605 |
| } |
| }, |
| "grouping": { |
| "max_questions_per_request": 1, |
| "by": "none (one question per request)" |
| }, |
| "environment": { |
| "threads": 4, |
| "machine": "arm64", |
| "system": "Darwin" |
| }, |
| "run": { |
| "checkpoint": "runs/r15-vision/stage2", |
| "data": "data/r12/cpu_latency.jsonl", |
| "data_sha256": "352f0b4676aeb7389b45a93086169096d6b8e3962568543cfa373877c3e84020", |
| "role": "unspecified", |
| "device": "cpu", |
| "temperature": "fitted", |
| "control": "none", |
| "seed": 0, |
| "code": "6feb0b3+dirty" |
| } |
| } |
