1gpu-llm-small-en-it-base / benchmark_metrics.json
nazdef's picture
Add files using upload-large-folder tool
0982b30 verified
Raw
History Blame Contribute Delete
21 kB
{
"checkpoints": {
"manual-manual-step_8050-8050": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8050",
"checkpoint_selector": "manual",
"checkpoint_step": 8050,
"cloze_en_contains": 0.04,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.16,
"cloze_it_exact": 0.0,
"distinct_1": 0.25430210325047803,
"distinct_2": 0.49204771371769385,
"language_consistency_en": 1.0,
"language_consistency_it": 0.8,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.475,
"ppl_en": 160.98727346833914,
"ppl_it": 49.3262033438898,
"ppl_mixed": 148.87818824189094,
"repeated_4gram_rate": 0.8,
"source_loss_books_en": 5.168394906180246,
"source_loss_books_it": 4.828146798270089,
"source_loss_code": 8.584260050455729,
"source_loss_web_en": 6.076142963610198,
"source_loss_web_it": 5.855331019351357,
"source_loss_wiki_en": 3.9519438310102983,
"source_loss_wiki_it": 3.720078430175781,
"val_loss_en": 5.081325315079599,
"val_loss_it": 3.8984554478379545,
"val_loss_mixed": 5.003128443008814
},
"manual-manual-step_8100-8100": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8100",
"checkpoint_selector": "manual",
"checkpoint_step": 8100,
"cloze_en_contains": 0.02,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.1,
"cloze_it_exact": 0.0,
"distinct_1": 0.24681684622918706,
"distinct_2": 0.47706422018348627,
"language_consistency_en": 0.975,
"language_consistency_it": 0.775,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.5,
"ppl_en": 133.99624078278714,
"ppl_it": 37.37222258583596,
"ppl_mixed": 146.63460264455082,
"repeated_4gram_rate": 0.875,
"source_loss_books_en": 5.14651852562314,
"source_loss_books_it": 4.574619838169643,
"source_loss_code": 8.119071960449219,
"source_loss_web_en": 5.854736328125,
"source_loss_web_it": 5.742109599866365,
"source_loss_wiki_en": 3.55948188088157,
"source_loss_wiki_it": 3.5610482788085935,
"val_loss_en": 4.897811745697597,
"val_loss_it": 3.620927716864914,
"val_loss_mixed": 4.987943796011118
},
"manual-manual-step_8150-8150": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8150",
"checkpoint_selector": "manual",
"checkpoint_step": 8150,
"cloze_en_contains": 0.0,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.12,
"cloze_it_exact": 0.0,
"distinct_1": 0.21788772597526165,
"distinct_2": 0.4065281899109792,
"language_consistency_en": 0.95,
"language_consistency_it": 0.875,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.55,
"ppl_en": 138.98176343604737,
"ppl_it": 40.75644648708747,
"ppl_mixed": 132.12770447884722,
"repeated_4gram_rate": 0.95,
"source_loss_books_en": 4.952012198311942,
"source_loss_books_it": 4.602359590076265,
"source_loss_code": 8.148141988118489,
"source_loss_web_en": 5.965742010819285,
"source_loss_web_it": 5.883667795281661,
"source_loss_wiki_en": 3.5794441916725854,
"source_loss_wiki_it": 3.5061331176757813,
"val_loss_en": 4.934342726221624,
"val_loss_it": 3.7076140231773502,
"val_loss_mixed": 4.883768913073418
},
"manual-manual-step_8200-8200": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8200",
"checkpoint_selector": "manual",
"checkpoint_step": 8200,
"cloze_en_contains": 0.06,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.2,
"cloze_it_exact": 0.0,
"distinct_1": 0.2344186046511628,
"distinct_2": 0.45120772946859905,
"language_consistency_en": 0.925,
"language_consistency_it": 0.775,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.55,
"ppl_en": 119.43683842254896,
"ppl_it": 39.62568688079664,
"ppl_mixed": 141.87928570121503,
"repeated_4gram_rate": 0.875,
"source_loss_books_en": 4.9834714617047995,
"source_loss_books_it": 4.592899140857515,
"source_loss_code": 7.946887715657552,
"source_loss_web_en": 5.851744802374589,
"source_loss_web_it": 5.77178955078125,
"source_loss_wiki_en": 3.5388662164861504,
"source_loss_wiki_it": 3.4404397583007813,
"val_loss_en": 4.782787682875147,
"val_loss_it": 3.679477566578349,
"val_loss_mixed": 4.954976595365084
},
"manual-manual-step_8250-8250": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8250",
"checkpoint_selector": "manual",
"checkpoint_step": 8250,
"cloze_en_contains": 0.06,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.12,
"cloze_it_exact": 0.0,
"distinct_1": 0.24417426545086118,
"distinct_2": 0.479957805907173,
"language_consistency_en": 0.975,
"language_consistency_it": 0.775,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.425,
"ppl_en": 133.04067942239274,
"ppl_it": 40.95397993234463,
"ppl_mixed": 128.22776351553605,
"repeated_4gram_rate": 0.85,
"source_loss_books_en": 5.1677569434756325,
"source_loss_books_it": 4.60965583437965,
"source_loss_code": 8.47701416015625,
"source_loss_web_en": 5.870997780247738,
"source_loss_web_it": 5.810668142218339,
"source_loss_wiki_en": 3.6867977489124644,
"source_loss_wiki_it": 3.667852478027344,
"val_loss_en": 4.8906549417747645,
"val_loss_it": 3.712448995621478,
"val_loss_mixed": 4.853808085123698
},
"manual-manual-step_8300-8300": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8300",
"checkpoint_selector": "manual",
"checkpoint_step": 8300,
"cloze_en_contains": 0.02,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.22,
"cloze_it_exact": 0.0,
"distinct_1": 0.2651821862348178,
"distinct_2": 0.5284810126582279,
"language_consistency_en": 0.925,
"language_consistency_it": 0.825,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.475,
"ppl_en": 122.53049941787616,
"ppl_it": 41.22488353010851,
"ppl_mixed": 130.1028710810426,
"repeated_4gram_rate": 0.75,
"source_loss_books_en": 4.971470424107143,
"source_loss_books_it": 4.578676859537761,
"source_loss_code": 8.002923075358073,
"source_loss_web_en": 5.837479039242393,
"source_loss_web_it": 5.753503899825247,
"source_loss_wiki_en": 3.497833251953125,
"source_loss_wiki_it": 3.5379019165039063,
"val_loss_en": 4.808359973835495,
"val_loss_it": 3.719042043216893,
"val_loss_mixed": 4.868325453538161
},
"manual-manual-step_8350-8350": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8350",
"checkpoint_selector": "manual",
"checkpoint_step": 8350,
"cloze_en_contains": 0.06,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.26,
"cloze_it_exact": 0.0,
"distinct_1": 0.24216524216524216,
"distinct_2": 0.47482724580454094,
"language_consistency_en": 0.95,
"language_consistency_it": 0.85,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.475,
"ppl_en": 119.40663408296695,
"ppl_it": 41.81570512528862,
"ppl_mixed": 127.06659304413428,
"repeated_4gram_rate": 0.9,
"source_loss_books_en": 4.8524594988141745,
"source_loss_books_it": 4.616578601655506,
"source_loss_code": 8.10670166015625,
"source_loss_web_en": 5.806581597579153,
"source_loss_web_it": 5.734786585757607,
"source_loss_wiki_en": 3.5233064131303267,
"source_loss_wiki_it": 3.5757290649414064,
"val_loss_en": 4.782534761248894,
"val_loss_it": 3.7332719896660476,
"val_loss_mixed": 4.8447113037109375
},
"manual-manual-step_8400-8400": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8400",
"checkpoint_selector": "manual",
"checkpoint_step": 8400,
"cloze_en_contains": 0.02,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.2,
"cloze_it_exact": 0.0,
"distinct_1": 0.23876404494382023,
"distinct_2": 0.4698443579766537,
"language_consistency_en": 0.975,
"language_consistency_it": 0.85,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.575,
"ppl_en": 121.76534720287194,
"ppl_it": 39.40941173051894,
"ppl_mixed": 130.8208191932363,
"repeated_4gram_rate": 0.95,
"source_loss_books_en": 4.808423723493304,
"source_loss_books_it": 4.581278846377418,
"source_loss_code": 8.033599344889323,
"source_loss_web_en": 5.863493266858552,
"source_loss_web_it": 5.778697766755757,
"source_loss_wiki_en": 3.4667070562189277,
"source_loss_wiki_it": 3.427510986328125,
"val_loss_en": 4.802095809072818,
"val_loss_it": 3.6740046641865716,
"val_loss_mixed": 4.873828594501202
},
"manual-manual-step_8450-8450": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8450",
"checkpoint_selector": "manual",
"checkpoint_step": 8450,
"cloze_en_contains": 0.08,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.22,
"cloze_it_exact": 0.0,
"distinct_1": 0.2554194156456173,
"distinct_2": 0.5142017629774731,
"language_consistency_en": 0.975,
"language_consistency_it": 0.85,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.35,
"ppl_en": 120.77124759532177,
"ppl_it": 39.07912028766744,
"ppl_mixed": 129.38063702618962,
"repeated_4gram_rate": 0.85,
"source_loss_books_en": 4.866651625860305,
"source_loss_books_it": 4.638146173386347,
"source_loss_code": 7.7475123087565105,
"source_loss_web_en": 5.8520764802631575,
"source_loss_web_it": 5.729297035618832,
"source_loss_wiki_en": 3.4290611960671167,
"source_loss_wiki_it": 3.497872314453125,
"val_loss_en": 4.793898240575251,
"val_loss_it": 3.665588316370229,
"val_loss_mixed": 4.86275873428736
},
"manual-manual-step_8500-8500": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8500",
"checkpoint_selector": "manual",
"checkpoint_step": 8500,
"cloze_en_contains": 0.06,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.24,
"cloze_it_exact": 0.0,
"distinct_1": 0.2557471264367816,
"distinct_2": 0.499003984063745,
"language_consistency_en": 0.975,
"language_consistency_it": 0.75,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.45,
"ppl_en": 121.22855762937996,
"ppl_it": 39.18571718583911,
"ppl_mixed": 130.34516505580527,
"repeated_4gram_rate": 0.775,
"source_loss_books_en": 4.840805780319941,
"source_loss_books_it": 4.553114028204055,
"source_loss_code": 7.899540710449219,
"source_loss_web_en": 5.917284513774671,
"source_loss_web_it": 5.725276746247944,
"source_loss_wiki_en": 3.5159357244318183,
"source_loss_wiki_it": 3.5039968872070313,
"val_loss_en": 4.797677669885023,
"val_loss_it": 3.668312322897989,
"val_loss_mixed": 4.870186047676282
},
"manual-manual-step_8550-8550": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8550",
"checkpoint_selector": "manual",
"checkpoint_step": 8550,
"cloze_en_contains": 0.06,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.18,
"cloze_it_exact": 0.0,
"distinct_1": 0.23950381679389313,
"distinct_2": 0.4722222222222222,
"language_consistency_en": 0.9650000000000001,
"language_consistency_it": 0.775,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.45,
"ppl_en": 122.47315286783923,
"ppl_it": 37.17054792048566,
"ppl_mixed": 123.7644678987954,
"repeated_4gram_rate": 0.925,
"source_loss_books_en": 4.8930544172014505,
"source_loss_books_it": 4.572469075520833,
"source_loss_code": 7.85228271484375,
"source_loss_web_en": 5.950106169048109,
"source_loss_web_it": 5.7168028982062085,
"source_loss_wiki_en": 3.4762639132413,
"source_loss_wiki_it": 3.4177459716796874,
"val_loss_en": 4.807891845703125,
"val_loss_it": 3.615516725133677,
"val_loss_mixed": 4.818380306928586
},
"manual-manual-step_8600-8600": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8600",
"checkpoint_selector": "manual",
"checkpoint_step": 8600,
"cloze_en_contains": 0.14,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.2,
"cloze_it_exact": 0.0,
"distinct_1": 0.22793404461687683,
"distinct_2": 0.4591321897073663,
"language_consistency_en": 0.975,
"language_consistency_it": 0.825,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.525,
"ppl_en": 122.42728790326866,
"ppl_it": 38.1497785499055,
"ppl_mixed": 121.06936368453971,
"repeated_4gram_rate": 0.925,
"source_loss_books_en": 4.86205073765346,
"source_loss_books_it": 4.5494671776181175,
"source_loss_code": 8.180701192220052,
"source_loss_web_en": 5.923584787469161,
"source_loss_web_it": 5.760439019454153,
"source_loss_wiki_en": 3.4361277493563565,
"source_loss_wiki_it": 3.4338027954101564,
"val_loss_en": 4.807517285616893,
"val_loss_it": 3.641519952992924,
"val_loss_mixed": 4.796363634940906
},
"manual-manual-step_8650-8650": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8650",
"checkpoint_selector": "manual",
"checkpoint_step": 8650,
"cloze_en_contains": 0.1,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.26,
"cloze_it_exact": 0.0,
"distinct_1": 0.2588686481303931,
"distinct_2": 0.5114656031904287,
"language_consistency_en": 0.9400000000000001,
"language_consistency_it": 0.825,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.5,
"ppl_en": 118.89374549864857,
"ppl_it": 37.72449273991971,
"ppl_mixed": 121.37686727143264,
"repeated_4gram_rate": 0.8,
"source_loss_books_en": 4.864224025181362,
"source_loss_books_it": 4.59336671375093,
"source_loss_code": 7.987039184570312,
"source_loss_web_en": 5.791411550421464,
"source_loss_web_it": 5.6818381861636515,
"source_loss_wiki_en": 3.469675237482244,
"source_loss_wiki_it": 3.5061785888671877,
"val_loss_en": 4.778230199274027,
"val_loss_it": 3.630309558305584,
"val_loss_mixed": 4.798900310809795
},
"manual-manual-step_8700-8700": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8700",
"checkpoint_selector": "manual",
"checkpoint_step": 8700,
"cloze_en_contains": 0.06,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.2,
"cloze_it_exact": 0.0,
"distinct_1": 0.2389463781749765,
"distinct_2": 0.46432062561094817,
"language_consistency_en": 0.975,
"language_consistency_it": 0.75,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.5,
"ppl_en": 117.21982774121467,
"ppl_it": 37.26508673165992,
"ppl_mixed": 128.9638946211249,
"repeated_4gram_rate": 0.9,
"source_loss_books_en": 4.919521150134859,
"source_loss_books_it": 4.648129781087239,
"source_loss_code": 8.037347412109375,
"source_loss_web_en": 5.900805423134251,
"source_loss_web_it": 5.770890888414885,
"source_loss_wiki_en": 3.4160707647150215,
"source_loss_wiki_it": 3.4827593994140624,
"val_loss_en": 4.764051041513119,
"val_loss_it": 3.618056875760438,
"val_loss_mixed": 4.859532478528145
},
"manual-manual-step_8750-8750": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8750",
"checkpoint_selector": "manual",
"checkpoint_step": 8750,
"cloze_en_contains": 0.04,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.2,
"cloze_it_exact": 0.0,
"distinct_1": 0.22843601895734597,
"distinct_2": 0.4630541871921182,
"language_consistency_en": 0.95,
"language_consistency_it": 0.775,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.475,
"ppl_en": 119.90702815340026,
"ppl_it": 37.823522345799645,
"ppl_mixed": 130.1233356475696,
"repeated_4gram_rate": 0.95,
"source_loss_books_en": 4.9090082077752974,
"source_loss_books_it": 4.570150466192336,
"source_loss_code": 8.192756652832031,
"source_loss_web_en": 5.895728261847245,
"source_loss_web_it": 5.751687300832648,
"source_loss_wiki_en": 3.460148898037997,
"source_loss_wiki_it": 3.5097088623046875,
"val_loss_en": 4.786716677107901,
"val_loss_it": 3.6329311933673796,
"val_loss_mixed": 4.868482736440805
},
"manual-manual-step_8800-8800": {
"aggregate_validation_loss_mean": null,
"aggregate_validation_perplexity_mean": null,
"checkpoint_name": "step_8800",
"checkpoint_selector": "manual",
"checkpoint_step": 8800,
"cloze_en_contains": 0.04,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.24,
"cloze_it_exact": 0.0,
"distinct_1": 0.25,
"distinct_2": 0.5088235294117647,
"language_consistency_en": 0.925,
"language_consistency_it": 0.8,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.375,
"ppl_en": 115.77590188514857,
"ppl_it": 37.72263850171775,
"ppl_mixed": 127.7439933507041,
"repeated_4gram_rate": 0.9,
"source_loss_books_en": 4.8956891014462425,
"source_loss_books_it": 4.594246273949032,
"source_loss_code": 8.056328837076823,
"source_loss_web_en": 5.855930930689762,
"source_loss_web_it": 5.714005721242804,
"source_loss_wiki_en": 3.47349028153853,
"source_loss_wiki_it": 3.541571044921875,
"val_loss_en": 4.751656442318323,
"val_loss_it": 3.630260404993276,
"val_loss_mixed": 4.850028209197215
}
},
"cloze_en_contains": 0.14,
"cloze_en_exact": 0.0,
"cloze_it_contains": 0.2,
"cloze_it_exact": 0.0,
"distinct_1": 0.22793404461687683,
"distinct_2": 0.4591321897073663,
"language_consistency_en": 0.975,
"language_consistency_it": 0.825,
"language_switch_rate_en": 0.0,
"language_switch_rate_it": 0.0,
"loop_rate": 0.525,
"ppl_en": 122.42728790326866,
"ppl_it": 38.1497785499055,
"ppl_mixed": 121.06936368453971,
"recommended_checkpoint": {
"checkpoint_name": "step_8600",
"checkpoint_path": "/mnt/apps/llm-nanochat/checkpoints/20260622_resume-gpt2small-gpt2preln-k20-wsds800-final2e5-webwiki-step8000-dense50/step_8600.pt",
"direction": "min",
"value": 4.796363634940906
},
"recommended_metric": "val_loss_mixed",
"repeated_4gram_rate": 0.925,
"source_losses": {
"books_en": 4.86205073765346,
"books_it": 4.5494671776181175,
"code": 8.180701192220052,
"web_en": 5.923584787469161,
"web_it": 5.760439019454153,
"wiki_en": 3.4361277493563565,
"wiki_it": 3.4338027954101564
},
"val_loss_en": 4.807517285616893,
"val_loss_it": 3.641519952992924,
"val_loss_mixed": 4.796363634940906
}