| { | |
| "protocol": "Local 1000-question stratified comparison, not official leaderboard scores; see README.md", | |
| "results": { | |
| "repair": { | |
| "model": "repair", | |
| "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", | |
| "n": 1000, | |
| "correct": 865, | |
| "errors": 0, | |
| "macro_accuracy": 0.8651282861809177, | |
| "tasks": { | |
| "ai2d": { | |
| "n": 38, | |
| "correct": 28, | |
| "accuracy": 0.7368421052631579, | |
| "errors": 0 | |
| }, | |
| "anli": { | |
| "n": 37, | |
| "correct": 24, | |
| "accuracy": 0.6486486486486487, | |
| "errors": 0 | |
| }, | |
| "aokvqa": { | |
| "n": 37, | |
| "correct": 33, | |
| "accuracy": 0.8918918918918919, | |
| "errors": 0 | |
| }, | |
| "boolq": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "corefud_de": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "corefud_en": { | |
| "n": 37, | |
| "correct": 29, | |
| "accuracy": 0.7837837837837838, | |
| "errors": 0 | |
| }, | |
| "docvqa_mc": { | |
| "n": 37, | |
| "correct": 36, | |
| "accuracy": 0.972972972972973, | |
| "errors": 0 | |
| }, | |
| "heldout_copa": { | |
| "n": 37, | |
| "correct": 35, | |
| "accuracy": 0.9459459459459459, | |
| "errors": 0 | |
| }, | |
| "heldout_wic": { | |
| "n": 37, | |
| "correct": 15, | |
| "accuracy": 0.40540540540540543, | |
| "errors": 0 | |
| }, | |
| "heldout_wsc.fixed": { | |
| "n": 37, | |
| "correct": 20, | |
| "accuracy": 0.5405405405405406, | |
| "errors": 0 | |
| }, | |
| "hellaswag": { | |
| "n": 37, | |
| "correct": 34, | |
| "accuracy": 0.918918918918919, | |
| "errors": 0 | |
| }, | |
| "iconqa": { | |
| "n": 37, | |
| "correct": 35, | |
| "accuracy": 0.9459459459459459, | |
| "errors": 0 | |
| }, | |
| "massive_de": { | |
| "n": 37, | |
| "correct": 30, | |
| "accuracy": 0.8108108108108109, | |
| "errors": 0 | |
| }, | |
| "pawsx_de": { | |
| "n": 37, | |
| "correct": 35, | |
| "accuracy": 0.9459459459459459, | |
| "errors": 0 | |
| }, | |
| "piqa": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "pope": { | |
| "n": 37, | |
| "correct": 35, | |
| "accuracy": 0.9459459459459459, | |
| "errors": 0 | |
| }, | |
| "scienceqa": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "social_iqa": { | |
| "n": 37, | |
| "correct": 27, | |
| "accuracy": 0.7297297297297297, | |
| "errors": 0 | |
| }, | |
| "sst2": { | |
| "n": 37, | |
| "correct": 34, | |
| "accuracy": 0.918918918918919, | |
| "errors": 0 | |
| }, | |
| "synthetic_dialogue": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "synthetic_numeric": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "synthetic_ocr": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "synthetic_rules": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "synthetic_temporal": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "textvqa_mc": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "winogrande": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "xnli_de": { | |
| "n": 37, | |
| "correct": 33, | |
| "accuracy": 0.8918918918918919, | |
| "errors": 0 | |
| } | |
| }, | |
| "modality": { | |
| "text": { | |
| "n": 703, | |
| "correct": 587, | |
| "mean_ms": 69.21716159307353, | |
| "median_ms": 65.98780000058468 | |
| }, | |
| "image": { | |
| "n": 297, | |
| "correct": 278, | |
| "mean_ms": 133.12665488233245, | |
| "median_ms": 130.3297999984352 | |
| } | |
| }, | |
| "inference_seconds": 88.19828109998343, | |
| "mean_ms": 88.19828109998343, | |
| "peak_vram_gib": 4.320731163024902, | |
| "peak_reserved_gib": 4.572265625, | |
| "wall_seconds": 101.53908681869507, | |
| "versions": { | |
| "torch": "2.11.0+cu128", | |
| "transformers": "5.17.0" | |
| } | |
| }, | |
| "bard": { | |
| "model": "bard", | |
| "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", | |
| "n": 1000, | |
| "correct": 809, | |
| "errors": 0, | |
| "macro_accuracy": 0.80891417733523, | |
| "tasks": { | |
| "ai2d": { | |
| "n": 38, | |
| "correct": 34, | |
| "accuracy": 0.8947368421052632, | |
| "errors": 0 | |
| }, | |
| "anli": { | |
| "n": 37, | |
| "correct": 21, | |
| "accuracy": 0.5675675675675675, | |
| "errors": 0 | |
| }, | |
| "aokvqa": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "boolq": { | |
| "n": 37, | |
| "correct": 33, | |
| "accuracy": 0.8918918918918919, | |
| "errors": 0 | |
| }, | |
| "corefud_de": { | |
| "n": 37, | |
| "correct": 21, | |
| "accuracy": 0.5675675675675675, | |
| "errors": 0 | |
| }, | |
| "corefud_en": { | |
| "n": 37, | |
| "correct": 29, | |
| "accuracy": 0.7837837837837838, | |
| "errors": 0 | |
| }, | |
| "docvqa_mc": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "heldout_copa": { | |
| "n": 37, | |
| "correct": 36, | |
| "accuracy": 0.972972972972973, | |
| "errors": 0 | |
| }, | |
| "heldout_wic": { | |
| "n": 37, | |
| "correct": 25, | |
| "accuracy": 0.6756756756756757, | |
| "errors": 0 | |
| }, | |
| "heldout_wsc.fixed": { | |
| "n": 37, | |
| "correct": 23, | |
| "accuracy": 0.6216216216216216, | |
| "errors": 0 | |
| }, | |
| "hellaswag": { | |
| "n": 37, | |
| "correct": 33, | |
| "accuracy": 0.8918918918918919, | |
| "errors": 0 | |
| }, | |
| "iconqa": { | |
| "n": 37, | |
| "correct": 33, | |
| "accuracy": 0.8918918918918919, | |
| "errors": 0 | |
| }, | |
| "massive_de": { | |
| "n": 37, | |
| "correct": 23, | |
| "accuracy": 0.6216216216216216, | |
| "errors": 0 | |
| }, | |
| "pawsx_de": { | |
| "n": 37, | |
| "correct": 21, | |
| "accuracy": 0.5675675675675675, | |
| "errors": 0 | |
| }, | |
| "piqa": { | |
| "n": 37, | |
| "correct": 30, | |
| "accuracy": 0.8108108108108109, | |
| "errors": 0 | |
| }, | |
| "pope": { | |
| "n": 37, | |
| "correct": 33, | |
| "accuracy": 0.8918918918918919, | |
| "errors": 0 | |
| }, | |
| "scienceqa": { | |
| "n": 37, | |
| "correct": 34, | |
| "accuracy": 0.918918918918919, | |
| "errors": 0 | |
| }, | |
| "social_iqa": { | |
| "n": 37, | |
| "correct": 27, | |
| "accuracy": 0.7297297297297297, | |
| "errors": 0 | |
| }, | |
| "sst2": { | |
| "n": 37, | |
| "correct": 34, | |
| "accuracy": 0.918918918918919, | |
| "errors": 0 | |
| }, | |
| "synthetic_dialogue": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "synthetic_numeric": { | |
| "n": 37, | |
| "correct": 22, | |
| "accuracy": 0.5945945945945946, | |
| "errors": 0 | |
| }, | |
| "synthetic_ocr": { | |
| "n": 37, | |
| "correct": 35, | |
| "accuracy": 0.9459459459459459, | |
| "errors": 0 | |
| }, | |
| "synthetic_rules": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "synthetic_temporal": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "textvqa_mc": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "winogrande": { | |
| "n": 37, | |
| "correct": 26, | |
| "accuracy": 0.7027027027027027, | |
| "errors": 0 | |
| }, | |
| "xnli_de": { | |
| "n": 37, | |
| "correct": 24, | |
| "accuracy": 0.6486486486486487, | |
| "errors": 0 | |
| } | |
| }, | |
| "modality": { | |
| "text": { | |
| "n": 703, | |
| "correct": 534, | |
| "mean_ms": 61.23855462305497, | |
| "median_ms": 53.49959999875864 | |
| }, | |
| "image": { | |
| "n": 297, | |
| "correct": 275, | |
| "mean_ms": 145.8211811447535, | |
| "median_ms": 139.9980999995023 | |
| } | |
| }, | |
| "inference_seconds": 86.35959469999943, | |
| "mean_ms": 86.35959469999943, | |
| "peak_vram_gib": 8.430227279663086, | |
| "peak_reserved_gib": 8.548828125, | |
| "wall_seconds": 98.90849375724792, | |
| "versions": { | |
| "torch": "2.11.0+cu128", | |
| "transformers": "4.57.3" | |
| } | |
| }, | |
| "diffusionvl": { | |
| "model": "diffusionvl", | |
| "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", | |
| "n": 1000, | |
| "correct": 737, | |
| "errors": 0, | |
| "macro_accuracy": 0.7369738159211844, | |
| "tasks": { | |
| "ai2d": { | |
| "n": 38, | |
| "correct": 29, | |
| "accuracy": 0.7631578947368421, | |
| "errors": 0 | |
| }, | |
| "anli": { | |
| "n": 37, | |
| "correct": 16, | |
| "accuracy": 0.43243243243243246, | |
| "errors": 0 | |
| }, | |
| "aokvqa": { | |
| "n": 37, | |
| "correct": 34, | |
| "accuracy": 0.918918918918919, | |
| "errors": 0 | |
| }, | |
| "boolq": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "corefud_de": { | |
| "n": 37, | |
| "correct": 18, | |
| "accuracy": 0.4864864864864865, | |
| "errors": 0 | |
| }, | |
| "corefud_en": { | |
| "n": 37, | |
| "correct": 19, | |
| "accuracy": 0.5135135135135135, | |
| "errors": 0 | |
| }, | |
| "docvqa_mc": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "heldout_copa": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "heldout_wic": { | |
| "n": 37, | |
| "correct": 20, | |
| "accuracy": 0.5405405405405406, | |
| "errors": 0 | |
| }, | |
| "heldout_wsc.fixed": { | |
| "n": 37, | |
| "correct": 17, | |
| "accuracy": 0.4594594594594595, | |
| "errors": 0 | |
| }, | |
| "hellaswag": { | |
| "n": 37, | |
| "correct": 25, | |
| "accuracy": 0.6756756756756757, | |
| "errors": 0 | |
| }, | |
| "iconqa": { | |
| "n": 37, | |
| "correct": 27, | |
| "accuracy": 0.7297297297297297, | |
| "errors": 0 | |
| }, | |
| "massive_de": { | |
| "n": 37, | |
| "correct": 21, | |
| "accuracy": 0.5675675675675675, | |
| "errors": 0 | |
| }, | |
| "pawsx_de": { | |
| "n": 37, | |
| "correct": 21, | |
| "accuracy": 0.5675675675675675, | |
| "errors": 0 | |
| }, | |
| "piqa": { | |
| "n": 37, | |
| "correct": 30, | |
| "accuracy": 0.8108108108108109, | |
| "errors": 0 | |
| }, | |
| "pope": { | |
| "n": 37, | |
| "correct": 34, | |
| "accuracy": 0.918918918918919, | |
| "errors": 0 | |
| }, | |
| "scienceqa": { | |
| "n": 37, | |
| "correct": 26, | |
| "accuracy": 0.7027027027027027, | |
| "errors": 0 | |
| }, | |
| "social_iqa": { | |
| "n": 37, | |
| "correct": 31, | |
| "accuracy": 0.8378378378378378, | |
| "errors": 0 | |
| }, | |
| "sst2": { | |
| "n": 37, | |
| "correct": 34, | |
| "accuracy": 0.918918918918919, | |
| "errors": 0 | |
| }, | |
| "synthetic_dialogue": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "synthetic_numeric": { | |
| "n": 37, | |
| "correct": 20, | |
| "accuracy": 0.5405405405405406, | |
| "errors": 0 | |
| }, | |
| "synthetic_ocr": { | |
| "n": 37, | |
| "correct": 33, | |
| "accuracy": 0.8918918918918919, | |
| "errors": 0 | |
| }, | |
| "synthetic_rules": { | |
| "n": 37, | |
| "correct": 35, | |
| "accuracy": 0.9459459459459459, | |
| "errors": 0 | |
| }, | |
| "synthetic_temporal": { | |
| "n": 37, | |
| "correct": 30, | |
| "accuracy": 0.8108108108108109, | |
| "errors": 0 | |
| }, | |
| "textvqa_mc": { | |
| "n": 37, | |
| "correct": 36, | |
| "accuracy": 0.972972972972973, | |
| "errors": 0 | |
| }, | |
| "winogrande": { | |
| "n": 37, | |
| "correct": 24, | |
| "accuracy": 0.6486486486486487, | |
| "errors": 0 | |
| }, | |
| "xnli_de": { | |
| "n": 37, | |
| "correct": 19, | |
| "accuracy": 0.5135135135135135, | |
| "errors": 0 | |
| } | |
| }, | |
| "modality": { | |
| "text": { | |
| "n": 703, | |
| "correct": 481, | |
| "mean_ms": 44.277235561917465, | |
| "median_ms": 40.86510000342969 | |
| }, | |
| "image": { | |
| "n": 297, | |
| "correct": 256, | |
| "mean_ms": 185.03554680136088, | |
| "median_ms": 177.97389999759616 | |
| } | |
| }, | |
| "inference_seconds": 86.08245400003216, | |
| "mean_ms": 86.08245400003216, | |
| "peak_vram_gib": 7.2419915199279785, | |
| "peak_reserved_gib": 7.55859375, | |
| "wall_seconds": 97.54556059837341, | |
| "versions": { | |
| "torch": "2.11.0+cu128", | |
| "transformers": "4.57.3" | |
| } | |
| }, | |
| "litjev4b": { | |
| "model": "litjev4b", | |
| "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", | |
| "n": 1000, | |
| "correct": 771, | |
| "errors": 0, | |
| "macro_accuracy": 0.7710341920868237, | |
| "tasks": { | |
| "ai2d": { | |
| "n": 38, | |
| "correct": 28, | |
| "accuracy": 0.7368421052631579, | |
| "errors": 0 | |
| }, | |
| "anli": { | |
| "n": 37, | |
| "correct": 15, | |
| "accuracy": 0.40540540540540543, | |
| "errors": 0 | |
| }, | |
| "aokvqa": { | |
| "n": 37, | |
| "correct": 34, | |
| "accuracy": 0.918918918918919, | |
| "errors": 0 | |
| }, | |
| "boolq": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "corefud_de": { | |
| "n": 37, | |
| "correct": 19, | |
| "accuracy": 0.5135135135135135, | |
| "errors": 0 | |
| }, | |
| "corefud_en": { | |
| "n": 37, | |
| "correct": 23, | |
| "accuracy": 0.6216216216216216, | |
| "errors": 0 | |
| }, | |
| "docvqa_mc": { | |
| "n": 37, | |
| "correct": 35, | |
| "accuracy": 0.9459459459459459, | |
| "errors": 0 | |
| }, | |
| "heldout_copa": { | |
| "n": 37, | |
| "correct": 35, | |
| "accuracy": 0.9459459459459459, | |
| "errors": 0 | |
| }, | |
| "heldout_wic": { | |
| "n": 37, | |
| "correct": 22, | |
| "accuracy": 0.5945945945945946, | |
| "errors": 0 | |
| }, | |
| "heldout_wsc.fixed": { | |
| "n": 37, | |
| "correct": 19, | |
| "accuracy": 0.5135135135135135, | |
| "errors": 0 | |
| }, | |
| "hellaswag": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "iconqa": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "massive_de": { | |
| "n": 37, | |
| "correct": 27, | |
| "accuracy": 0.7297297297297297, | |
| "errors": 0 | |
| }, | |
| "pawsx_de": { | |
| "n": 37, | |
| "correct": 26, | |
| "accuracy": 0.7027027027027027, | |
| "errors": 0 | |
| }, | |
| "piqa": { | |
| "n": 37, | |
| "correct": 27, | |
| "accuracy": 0.7297297297297297, | |
| "errors": 0 | |
| }, | |
| "pope": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "scienceqa": { | |
| "n": 37, | |
| "correct": 36, | |
| "accuracy": 0.972972972972973, | |
| "errors": 0 | |
| }, | |
| "social_iqa": { | |
| "n": 37, | |
| "correct": 27, | |
| "accuracy": 0.7297297297297297, | |
| "errors": 0 | |
| }, | |
| "sst2": { | |
| "n": 37, | |
| "correct": 36, | |
| "accuracy": 0.972972972972973, | |
| "errors": 0 | |
| }, | |
| "synthetic_dialogue": { | |
| "n": 37, | |
| "correct": 34, | |
| "accuracy": 0.918918918918919, | |
| "errors": 0 | |
| }, | |
| "synthetic_numeric": { | |
| "n": 37, | |
| "correct": 19, | |
| "accuracy": 0.5135135135135135, | |
| "errors": 0 | |
| }, | |
| "synthetic_ocr": { | |
| "n": 37, | |
| "correct": 36, | |
| "accuracy": 0.972972972972973, | |
| "errors": 0 | |
| }, | |
| "synthetic_rules": { | |
| "n": 37, | |
| "correct": 34, | |
| "accuracy": 0.918918918918919, | |
| "errors": 0 | |
| }, | |
| "synthetic_temporal": { | |
| "n": 37, | |
| "correct": 32, | |
| "accuracy": 0.8648648648648649, | |
| "errors": 0 | |
| }, | |
| "textvqa_mc": { | |
| "n": 37, | |
| "correct": 37, | |
| "accuracy": 1.0, | |
| "errors": 0 | |
| }, | |
| "winogrande": { | |
| "n": 37, | |
| "correct": 20, | |
| "accuracy": 0.5405405405405406, | |
| "errors": 0 | |
| }, | |
| "xnli_de": { | |
| "n": 37, | |
| "correct": 22, | |
| "accuracy": 0.5945945945945946, | |
| "errors": 0 | |
| } | |
| }, | |
| "modality": { | |
| "text": { | |
| "n": 703, | |
| "correct": 501, | |
| "mean_ms": 136.2282305831357, | |
| "median_ms": 126.98180000006687 | |
| }, | |
| "image": { | |
| "n": 297, | |
| "correct": 270, | |
| "mean_ms": 228.37314949524182, | |
| "median_ms": 221.82390000671148 | |
| } | |
| }, | |
| "inference_seconds": 163.59527150003123, | |
| "mean_ms": 163.59527150003123, | |
| "peak_vram_gib": 8.78392219543457, | |
| "peak_reserved_gib": 8.935546875, | |
| "wall_seconds": 181.67939114570618, | |
| "versions": { | |
| "torch": "2.11.0+cu128", | |
| "transformers": "5.17.0" | |
| } | |
| }, | |
| "base": { | |
| "model": "base", | |
| "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", | |
| "n": 1000, | |
| "correct": 700, | |
| "errors": 0, | |
| "macro_accuracy": 0.7000421474105685, | |
| "tasks": { | |
| "ai2d": { | |
| "n": 38, | |
| "correct": 25, | |
| "accuracy": 0.6578947368421053, | |
| "errors": 0 | |
| }, | |
| "anli": { | |
| "n": 37, | |
| "correct": 19, | |
| "accuracy": 0.5135135135135135, | |
| "errors": 0 | |
| }, | |
| "aokvqa": { | |
| "n": 37, | |
| "correct": 25, | |
| "accuracy": 0.6756756756756757, | |
| "errors": 0 | |
| }, | |
| "boolq": { | |
| "n": 37, | |
| "correct": 30, | |
| "accuracy": 0.8108108108108109, | |
| "errors": 0 | |
| }, | |
| "corefud_de": { | |
| "n": 37, | |
| "correct": 19, | |
| "accuracy": 0.5135135135135135, | |
| "errors": 0 | |
| }, | |
| "corefud_en": { | |
| "n": 37, | |
| "correct": 15, | |
| "accuracy": 0.40540540540540543, | |
| "errors": 0 | |
| }, | |
| "docvqa_mc": { | |
| "n": 37, | |
| "correct": 36, | |
| "accuracy": 0.972972972972973, | |
| "errors": 0 | |
| }, | |
| "heldout_copa": { | |
| "n": 37, | |
| "correct": 31, | |
| "accuracy": 0.8378378378378378, | |
| "errors": 0 | |
| }, | |
| "heldout_wic": { | |
| "n": 37, | |
| "correct": 23, | |
| "accuracy": 0.6216216216216216, | |
| "errors": 0 | |
| }, | |
| "heldout_wsc.fixed": { | |
| "n": 37, | |
| "correct": 24, | |
| "accuracy": 0.6486486486486487, | |
| "errors": 0 | |
| }, | |
| "hellaswag": { | |
| "n": 37, | |
| "correct": 23, | |
| "accuracy": 0.6216216216216216, | |
| "errors": 0 | |
| }, | |
| "iconqa": { | |
| "n": 37, | |
| "correct": 27, | |
| "accuracy": 0.7297297297297297, | |
| "errors": 0 | |
| }, | |
| "massive_de": { | |
| "n": 37, | |
| "correct": 22, | |
| "accuracy": 0.5945945945945946, | |
| "errors": 0 | |
| }, | |
| "pawsx_de": { | |
| "n": 37, | |
| "correct": 21, | |
| "accuracy": 0.5675675675675675, | |
| "errors": 0 | |
| }, | |
| "piqa": { | |
| "n": 37, | |
| "correct": 29, | |
| "accuracy": 0.7837837837837838, | |
| "errors": 0 | |
| }, | |
| "pope": { | |
| "n": 37, | |
| "correct": 33, | |
| "accuracy": 0.8918918918918919, | |
| "errors": 0 | |
| }, | |
| "scienceqa": { | |
| "n": 37, | |
| "correct": 30, | |
| "accuracy": 0.8108108108108109, | |
| "errors": 0 | |
| }, | |
| "social_iqa": { | |
| "n": 37, | |
| "correct": 25, | |
| "accuracy": 0.6756756756756757, | |
| "errors": 0 | |
| }, | |
| "sst2": { | |
| "n": 37, | |
| "correct": 34, | |
| "accuracy": 0.918918918918919, | |
| "errors": 0 | |
| }, | |
| "synthetic_dialogue": { | |
| "n": 37, | |
| "correct": 28, | |
| "accuracy": 0.7567567567567568, | |
| "errors": 0 | |
| }, | |
| "synthetic_numeric": { | |
| "n": 37, | |
| "correct": 16, | |
| "accuracy": 0.43243243243243246, | |
| "errors": 0 | |
| }, | |
| "synthetic_ocr": { | |
| "n": 37, | |
| "correct": 29, | |
| "accuracy": 0.7837837837837838, | |
| "errors": 0 | |
| }, | |
| "synthetic_rules": { | |
| "n": 37, | |
| "correct": 31, | |
| "accuracy": 0.8378378378378378, | |
| "errors": 0 | |
| }, | |
| "synthetic_temporal": { | |
| "n": 37, | |
| "correct": 27, | |
| "accuracy": 0.7297297297297297, | |
| "errors": 0 | |
| }, | |
| "textvqa_mc": { | |
| "n": 37, | |
| "correct": 36, | |
| "accuracy": 0.972972972972973, | |
| "errors": 0 | |
| }, | |
| "winogrande": { | |
| "n": 37, | |
| "correct": 26, | |
| "accuracy": 0.7027027027027027, | |
| "errors": 0 | |
| }, | |
| "xnli_de": { | |
| "n": 37, | |
| "correct": 16, | |
| "accuracy": 0.43243243243243246, | |
| "errors": 0 | |
| } | |
| }, | |
| "modality": { | |
| "text": { | |
| "n": 703, | |
| "correct": 459, | |
| "mean_ms": 46.8396625885729, | |
| "median_ms": 43.49789999832865 | |
| }, | |
| "image": { | |
| "n": 297, | |
| "correct": 241, | |
| "mean_ms": 106.90101212150651, | |
| "median_ms": 102.31720001320355 | |
| } | |
| }, | |
| "inference_seconds": 64.67788339985418, | |
| "mean_ms": 64.67788339985418, | |
| "peak_vram_gib": 4.258074760437012, | |
| "peak_reserved_gib": 4.46875, | |
| "wall_seconds": 76.89548850059509, | |
| "versions": { | |
| "torch": "2.11.0+cu128", | |
| "transformers": "5.17.0" | |
| } | |
| } | |
| }, | |
| "base_comparison_provenance": { | |
| "base_model": "Qwen/Qwen3.5-2B", | |
| "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", | |
| "adapter": false, | |
| "script_sha256": "a4f2a1be395925b0aa08a6cfa49fb1ab5e0dd820ec23afa165d66fd1d92285a7", | |
| "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", | |
| "timing_note": "Standalone base-only repeat with no concurrent model inference. Same 1000 questions, warmup, preprocessing and timing boundaries as the original comparison; ordinary desktop load remains uncontrolled.", | |
| "protocol": "Same MultiDecider, pinned 1000 rows, option order and image settings as repair; adapter=False. No tuning or calibration.", | |
| "prediction_reproducibility": "All 1000 selected answers exactly match the earlier base evaluation." | |
| }, | |
| "paired_adapter_vs_base": { | |
| "adapter_only": 224, | |
| "base_only": 59 | |
| } | |
| } | |