{ "protocol": "Local 1000-question stratified comparison, not official leaderboard scores; see README.md", "results": { "repair": { "model": "repair", "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", "n": 1000, "correct": 865, "errors": 0, "macro_accuracy": 0.8651282861809177, "tasks": { "ai2d": { "n": 38, "correct": 28, "accuracy": 0.7368421052631579, "errors": 0 }, "anli": { "n": 37, "correct": 24, "accuracy": 0.6486486486486487, "errors": 0 }, "aokvqa": { "n": 37, "correct": 33, "accuracy": 0.8918918918918919, "errors": 0 }, "boolq": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "corefud_de": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "corefud_en": { "n": 37, "correct": 29, "accuracy": 0.7837837837837838, "errors": 0 }, "docvqa_mc": { "n": 37, "correct": 36, "accuracy": 0.972972972972973, "errors": 0 }, "heldout_copa": { "n": 37, "correct": 35, "accuracy": 0.9459459459459459, "errors": 0 }, "heldout_wic": { "n": 37, "correct": 15, "accuracy": 0.40540540540540543, "errors": 0 }, "heldout_wsc.fixed": { "n": 37, "correct": 20, "accuracy": 0.5405405405405406, "errors": 0 }, "hellaswag": { "n": 37, "correct": 34, "accuracy": 0.918918918918919, "errors": 0 }, "iconqa": { "n": 37, "correct": 35, "accuracy": 0.9459459459459459, "errors": 0 }, "massive_de": { "n": 37, "correct": 30, "accuracy": 0.8108108108108109, "errors": 0 }, "pawsx_de": { "n": 37, "correct": 35, "accuracy": 0.9459459459459459, "errors": 0 }, "piqa": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "pope": { "n": 37, "correct": 35, "accuracy": 0.9459459459459459, "errors": 0 }, "scienceqa": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "social_iqa": { "n": 37, "correct": 27, "accuracy": 0.7297297297297297, "errors": 0 }, "sst2": { "n": 37, "correct": 34, "accuracy": 0.918918918918919, "errors": 0 }, "synthetic_dialogue": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "synthetic_numeric": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "synthetic_ocr": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "synthetic_rules": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "synthetic_temporal": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "textvqa_mc": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "winogrande": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "xnli_de": { "n": 37, "correct": 33, "accuracy": 0.8918918918918919, "errors": 0 } }, "modality": { "text": { "n": 703, "correct": 587, "mean_ms": 69.21716159307353, "median_ms": 65.98780000058468 }, "image": { "n": 297, "correct": 278, "mean_ms": 133.12665488233245, "median_ms": 130.3297999984352 } }, "inference_seconds": 88.19828109998343, "mean_ms": 88.19828109998343, "peak_vram_gib": 4.320731163024902, "peak_reserved_gib": 4.572265625, "wall_seconds": 101.53908681869507, "versions": { "torch": "2.11.0+cu128", "transformers": "5.17.0" } }, "bard": { "model": "bard", "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", "n": 1000, "correct": 809, "errors": 0, "macro_accuracy": 0.80891417733523, "tasks": { "ai2d": { "n": 38, "correct": 34, "accuracy": 0.8947368421052632, "errors": 0 }, "anli": { "n": 37, "correct": 21, "accuracy": 0.5675675675675675, "errors": 0 }, "aokvqa": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "boolq": { "n": 37, "correct": 33, "accuracy": 0.8918918918918919, "errors": 0 }, "corefud_de": { "n": 37, "correct": 21, "accuracy": 0.5675675675675675, "errors": 0 }, "corefud_en": { "n": 37, "correct": 29, "accuracy": 0.7837837837837838, "errors": 0 }, "docvqa_mc": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "heldout_copa": { "n": 37, "correct": 36, "accuracy": 0.972972972972973, "errors": 0 }, "heldout_wic": { "n": 37, "correct": 25, "accuracy": 0.6756756756756757, "errors": 0 }, "heldout_wsc.fixed": { "n": 37, "correct": 23, "accuracy": 0.6216216216216216, "errors": 0 }, "hellaswag": { "n": 37, "correct": 33, "accuracy": 0.8918918918918919, "errors": 0 }, "iconqa": { "n": 37, "correct": 33, "accuracy": 0.8918918918918919, "errors": 0 }, "massive_de": { "n": 37, "correct": 23, "accuracy": 0.6216216216216216, "errors": 0 }, "pawsx_de": { "n": 37, "correct": 21, "accuracy": 0.5675675675675675, "errors": 0 }, "piqa": { "n": 37, "correct": 30, "accuracy": 0.8108108108108109, "errors": 0 }, "pope": { "n": 37, "correct": 33, "accuracy": 0.8918918918918919, "errors": 0 }, "scienceqa": { "n": 37, "correct": 34, "accuracy": 0.918918918918919, "errors": 0 }, "social_iqa": { "n": 37, "correct": 27, "accuracy": 0.7297297297297297, "errors": 0 }, "sst2": { "n": 37, "correct": 34, "accuracy": 0.918918918918919, "errors": 0 }, "synthetic_dialogue": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "synthetic_numeric": { "n": 37, "correct": 22, "accuracy": 0.5945945945945946, "errors": 0 }, "synthetic_ocr": { "n": 37, "correct": 35, "accuracy": 0.9459459459459459, "errors": 0 }, "synthetic_rules": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "synthetic_temporal": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "textvqa_mc": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "winogrande": { "n": 37, "correct": 26, "accuracy": 0.7027027027027027, "errors": 0 }, "xnli_de": { "n": 37, "correct": 24, "accuracy": 0.6486486486486487, "errors": 0 } }, "modality": { "text": { "n": 703, "correct": 534, "mean_ms": 61.23855462305497, "median_ms": 53.49959999875864 }, "image": { "n": 297, "correct": 275, "mean_ms": 145.8211811447535, "median_ms": 139.9980999995023 } }, "inference_seconds": 86.35959469999943, "mean_ms": 86.35959469999943, "peak_vram_gib": 8.430227279663086, "peak_reserved_gib": 8.548828125, "wall_seconds": 98.90849375724792, "versions": { "torch": "2.11.0+cu128", "transformers": "4.57.3" } }, "diffusionvl": { "model": "diffusionvl", "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", "n": 1000, "correct": 737, "errors": 0, "macro_accuracy": 0.7369738159211844, "tasks": { "ai2d": { "n": 38, "correct": 29, "accuracy": 0.7631578947368421, "errors": 0 }, "anli": { "n": 37, "correct": 16, "accuracy": 0.43243243243243246, "errors": 0 }, "aokvqa": { "n": 37, "correct": 34, "accuracy": 0.918918918918919, "errors": 0 }, "boolq": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "corefud_de": { "n": 37, "correct": 18, "accuracy": 0.4864864864864865, "errors": 0 }, "corefud_en": { "n": 37, "correct": 19, "accuracy": 0.5135135135135135, "errors": 0 }, "docvqa_mc": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "heldout_copa": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "heldout_wic": { "n": 37, "correct": 20, "accuracy": 0.5405405405405406, "errors": 0 }, "heldout_wsc.fixed": { "n": 37, "correct": 17, "accuracy": 0.4594594594594595, "errors": 0 }, "hellaswag": { "n": 37, "correct": 25, "accuracy": 0.6756756756756757, "errors": 0 }, "iconqa": { "n": 37, "correct": 27, "accuracy": 0.7297297297297297, "errors": 0 }, "massive_de": { "n": 37, "correct": 21, "accuracy": 0.5675675675675675, "errors": 0 }, "pawsx_de": { "n": 37, "correct": 21, "accuracy": 0.5675675675675675, "errors": 0 }, "piqa": { "n": 37, "correct": 30, "accuracy": 0.8108108108108109, "errors": 0 }, "pope": { "n": 37, "correct": 34, "accuracy": 0.918918918918919, "errors": 0 }, "scienceqa": { "n": 37, "correct": 26, "accuracy": 0.7027027027027027, "errors": 0 }, "social_iqa": { "n": 37, "correct": 31, "accuracy": 0.8378378378378378, "errors": 0 }, "sst2": { "n": 37, "correct": 34, "accuracy": 0.918918918918919, "errors": 0 }, "synthetic_dialogue": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "synthetic_numeric": { "n": 37, "correct": 20, "accuracy": 0.5405405405405406, "errors": 0 }, "synthetic_ocr": { "n": 37, "correct": 33, "accuracy": 0.8918918918918919, "errors": 0 }, "synthetic_rules": { "n": 37, "correct": 35, "accuracy": 0.9459459459459459, "errors": 0 }, "synthetic_temporal": { "n": 37, "correct": 30, "accuracy": 0.8108108108108109, "errors": 0 }, "textvqa_mc": { "n": 37, "correct": 36, "accuracy": 0.972972972972973, "errors": 0 }, "winogrande": { "n": 37, "correct": 24, "accuracy": 0.6486486486486487, "errors": 0 }, "xnli_de": { "n": 37, "correct": 19, "accuracy": 0.5135135135135135, "errors": 0 } }, "modality": { "text": { "n": 703, "correct": 481, "mean_ms": 44.277235561917465, "median_ms": 40.86510000342969 }, "image": { "n": 297, "correct": 256, "mean_ms": 185.03554680136088, "median_ms": 177.97389999759616 } }, "inference_seconds": 86.08245400003216, "mean_ms": 86.08245400003216, "peak_vram_gib": 7.2419915199279785, "peak_reserved_gib": 7.55859375, "wall_seconds": 97.54556059837341, "versions": { "torch": "2.11.0+cu128", "transformers": "4.57.3" } }, "litjev4b": { "model": "litjev4b", "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", "n": 1000, "correct": 771, "errors": 0, "macro_accuracy": 0.7710341920868237, "tasks": { "ai2d": { "n": 38, "correct": 28, "accuracy": 0.7368421052631579, "errors": 0 }, "anli": { "n": 37, "correct": 15, "accuracy": 0.40540540540540543, "errors": 0 }, "aokvqa": { "n": 37, "correct": 34, "accuracy": 0.918918918918919, "errors": 0 }, "boolq": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "corefud_de": { "n": 37, "correct": 19, "accuracy": 0.5135135135135135, "errors": 0 }, "corefud_en": { "n": 37, "correct": 23, "accuracy": 0.6216216216216216, "errors": 0 }, "docvqa_mc": { "n": 37, "correct": 35, "accuracy": 0.9459459459459459, "errors": 0 }, "heldout_copa": { "n": 37, "correct": 35, "accuracy": 0.9459459459459459, "errors": 0 }, "heldout_wic": { "n": 37, "correct": 22, "accuracy": 0.5945945945945946, "errors": 0 }, "heldout_wsc.fixed": { "n": 37, "correct": 19, "accuracy": 0.5135135135135135, "errors": 0 }, "hellaswag": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "iconqa": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "massive_de": { "n": 37, "correct": 27, "accuracy": 0.7297297297297297, "errors": 0 }, "pawsx_de": { "n": 37, "correct": 26, "accuracy": 0.7027027027027027, "errors": 0 }, "piqa": { "n": 37, "correct": 27, "accuracy": 0.7297297297297297, "errors": 0 }, "pope": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "scienceqa": { "n": 37, "correct": 36, "accuracy": 0.972972972972973, "errors": 0 }, "social_iqa": { "n": 37, "correct": 27, "accuracy": 0.7297297297297297, "errors": 0 }, "sst2": { "n": 37, "correct": 36, "accuracy": 0.972972972972973, "errors": 0 }, "synthetic_dialogue": { "n": 37, "correct": 34, "accuracy": 0.918918918918919, "errors": 0 }, "synthetic_numeric": { "n": 37, "correct": 19, "accuracy": 0.5135135135135135, "errors": 0 }, "synthetic_ocr": { "n": 37, "correct": 36, "accuracy": 0.972972972972973, "errors": 0 }, "synthetic_rules": { "n": 37, "correct": 34, "accuracy": 0.918918918918919, "errors": 0 }, "synthetic_temporal": { "n": 37, "correct": 32, "accuracy": 0.8648648648648649, "errors": 0 }, "textvqa_mc": { "n": 37, "correct": 37, "accuracy": 1.0, "errors": 0 }, "winogrande": { "n": 37, "correct": 20, "accuracy": 0.5405405405405406, "errors": 0 }, "xnli_de": { "n": 37, "correct": 22, "accuracy": 0.5945945945945946, "errors": 0 } }, "modality": { "text": { "n": 703, "correct": 501, "mean_ms": 136.2282305831357, "median_ms": 126.98180000006687 }, "image": { "n": 297, "correct": 270, "mean_ms": 228.37314949524182, "median_ms": 221.82390000671148 } }, "inference_seconds": 163.59527150003123, "mean_ms": 163.59527150003123, "peak_vram_gib": 8.78392219543457, "peak_reserved_gib": 8.935546875, "wall_seconds": 181.67939114570618, "versions": { "torch": "2.11.0+cu128", "transformers": "5.17.0" } }, "base": { "model": "base", "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", "n": 1000, "correct": 700, "errors": 0, "macro_accuracy": 0.7000421474105685, "tasks": { "ai2d": { "n": 38, "correct": 25, "accuracy": 0.6578947368421053, "errors": 0 }, "anli": { "n": 37, "correct": 19, "accuracy": 0.5135135135135135, "errors": 0 }, "aokvqa": { "n": 37, "correct": 25, "accuracy": 0.6756756756756757, "errors": 0 }, "boolq": { "n": 37, "correct": 30, "accuracy": 0.8108108108108109, "errors": 0 }, "corefud_de": { "n": 37, "correct": 19, "accuracy": 0.5135135135135135, "errors": 0 }, "corefud_en": { "n": 37, "correct": 15, "accuracy": 0.40540540540540543, "errors": 0 }, "docvqa_mc": { "n": 37, "correct": 36, "accuracy": 0.972972972972973, "errors": 0 }, "heldout_copa": { "n": 37, "correct": 31, "accuracy": 0.8378378378378378, "errors": 0 }, "heldout_wic": { "n": 37, "correct": 23, "accuracy": 0.6216216216216216, "errors": 0 }, "heldout_wsc.fixed": { "n": 37, "correct": 24, "accuracy": 0.6486486486486487, "errors": 0 }, "hellaswag": { "n": 37, "correct": 23, "accuracy": 0.6216216216216216, "errors": 0 }, "iconqa": { "n": 37, "correct": 27, "accuracy": 0.7297297297297297, "errors": 0 }, "massive_de": { "n": 37, "correct": 22, "accuracy": 0.5945945945945946, "errors": 0 }, "pawsx_de": { "n": 37, "correct": 21, "accuracy": 0.5675675675675675, "errors": 0 }, "piqa": { "n": 37, "correct": 29, "accuracy": 0.7837837837837838, "errors": 0 }, "pope": { "n": 37, "correct": 33, "accuracy": 0.8918918918918919, "errors": 0 }, "scienceqa": { "n": 37, "correct": 30, "accuracy": 0.8108108108108109, "errors": 0 }, "social_iqa": { "n": 37, "correct": 25, "accuracy": 0.6756756756756757, "errors": 0 }, "sst2": { "n": 37, "correct": 34, "accuracy": 0.918918918918919, "errors": 0 }, "synthetic_dialogue": { "n": 37, "correct": 28, "accuracy": 0.7567567567567568, "errors": 0 }, "synthetic_numeric": { "n": 37, "correct": 16, "accuracy": 0.43243243243243246, "errors": 0 }, "synthetic_ocr": { "n": 37, "correct": 29, "accuracy": 0.7837837837837838, "errors": 0 }, "synthetic_rules": { "n": 37, "correct": 31, "accuracy": 0.8378378378378378, "errors": 0 }, "synthetic_temporal": { "n": 37, "correct": 27, "accuracy": 0.7297297297297297, "errors": 0 }, "textvqa_mc": { "n": 37, "correct": 36, "accuracy": 0.972972972972973, "errors": 0 }, "winogrande": { "n": 37, "correct": 26, "accuracy": 0.7027027027027027, "errors": 0 }, "xnli_de": { "n": 37, "correct": 16, "accuracy": 0.43243243243243246, "errors": 0 } }, "modality": { "text": { "n": 703, "correct": 459, "mean_ms": 46.8396625885729, "median_ms": 43.49789999832865 }, "image": { "n": 297, "correct": 241, "mean_ms": 106.90101212150651, "median_ms": 102.31720001320355 } }, "inference_seconds": 64.67788339985418, "mean_ms": 64.67788339985418, "peak_vram_gib": 4.258074760437012, "peak_reserved_gib": 4.46875, "wall_seconds": 76.89548850059509, "versions": { "torch": "2.11.0+cu128", "transformers": "5.17.0" } } }, "base_comparison_provenance": { "base_model": "Qwen/Qwen3.5-2B", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "adapter": false, "script_sha256": "a4f2a1be395925b0aa08a6cfa49fb1ab5e0dd820ec23afa165d66fd1d92285a7", "dataset_sha256": "88cc14e7201b39b16545cd1e7578358a1747d528138d458c4351b2d27ed4e974", "timing_note": "Standalone base-only repeat with no concurrent model inference. Same 1000 questions, warmup, preprocessing and timing boundaries as the original comparison; ordinary desktop load remains uncontrolled.", "protocol": "Same MultiDecider, pinned 1000 rows, option order and image settings as repair; adapter=False. No tuning or calibration.", "prediction_reproducibility": "All 1000 selected answers exactly match the earlier base evaluation." }, "paired_adapter_vs_base": { "adapter_only": 224, "base_only": 59 } }