Upload blimp_all_models_comparison_en.csv

blimp_all_models_comparison_en.csv

@@ -0,0 +1,71 @@
+paradigm,Gen-3.6 (13.9M),GPT-2-large (774M),Transformer-XL (~257M),LSTM LM (Gulordava et al.),5-gram LM,Human (individual),gen36_beats_gpt2_large,gen36_beats_transformer_xl,gen36_beats_lstm,gen36_beats_ngram_5gram,gen36_beats_human
+adjunct_island,73.3%,89.4%,64.5%,67.4%,47.9%,94.1%,,YES,YES,YES,
+anaphor_gender_agreement,83.3%,99.4%,91.2%,88.2%,43.9%,96.0%,,,,YES,
+anaphor_number_agreement,86.7%,99.2%,97.0%,95.2%,51.9%,99.0%,,,,YES,
+animate_subject_passive,60.0%,76.6%,74.3%,71.5%,70.3%,86.1%,,,,,
+animate_subject_trans,76.7%,84.9%,88.5%,86.8%,90.7%,87.0%,,,,,
+causative,76.7%,78.2%,57.5%,68.4%,53.5%,98.0%,,YES,YES,YES,
+complex_NP_island,53.3%,72.2%,57.5%,47.4%,49.7%,80.0%,,,YES,YES,
+coordinate_structure_constraint_complex_left_branch,20.0%,81.0%,35.8%,30.1%,32.0%,90.0%,,,,,
+coordinate_structure_constraint_object_extraction,60.0%,85.4%,74.4%,70.5%,59.3%,90.9%,,,,YES,
+determiner_noun_agreement_1,93.3%,98.8%,91.7%,91.8%,88.3%,95.0%,,YES,YES,YES,
+determiner_noun_agreement_2,100.0%,97.7%,81.1%,91.9%,86.3%,95.0%,YES,YES,YES,YES,YES
+determiner_noun_agreement_irregular_1,76.7%,95.7%,88.3%,81.5%,85.4%,92.0%,,,,,
+determiner_noun_agreement_irregular_2,80.0%,95.3%,81.6%,86.0%,90.4%,84.2%,,,,,
+determiner_noun_agreement_with_adj_2,90.0%,94.9%,78.1%,86.2%,49.7%,96.0%,,YES,YES,YES,
+determiner_noun_agreement_with_adj_irregular_1,80.0%,92.7%,80.8%,76.2%,53.0%,94.0%,,,YES,YES,
+determiner_noun_agreement_with_adj_irregular_2,76.7%,94.0%,76.6%,82.9%,55.4%,85.0%,,YES,,YES,
+determiner_noun_agreement_with_adjective_1,80.0%,97.5%,85.9%,86.7%,51.9%,95.0%,,,,YES,
+distractor_agreement_relational_noun,73.3%,79.5%,77.2%,76.0%,23.5%,81.0%,,,,YES,
+distractor_agreement_relative_clause,50.0%,65.7%,59.8%,62.7%,21.7%,86.0%,,,,YES,
+drop_argument,66.7%,80.7%,70.3%,78.9%,71.6%,85.1%,,,,,
+ellipsis_n_bar_1,53.3%,91.5%,65.1%,67.8%,23.4%,91.1%,,,,YES,
+ellipsis_n_bar_2,96.7%,87.2%,89.2%,67.4%,50.4%,78.0%,YES,YES,YES,YES,YES
+existential_there_object_raising,60.0%,78.4%,75.9%,65.9%,83.5%,90.0%,,,,,
+existential_there_quantifiers_1,100.0%,99.5%,93.7%,96.3%,90.5%,94.0%,YES,YES,YES,YES,YES
+existential_there_quantifiers_2,13.3%,42.4%,14.0%,16.2%,62.3%,76.0%,,,,,
+existential_there_subject_raising,73.3%,91.1%,78.7%,79.5%,77.4%,88.0%,,,,,
+expletive_it_object_raising,66.7%,79.2%,71.5%,63.0%,71.5%,86.0%,,,YES,,
+inchoative,53.3%,65.9%,54.3%,65.3%,50.9%,82.0%,,,,YES,
+intransitive,63.3%,84.2%,67.2%,78.5%,67.8%,90.0%,,,,,
+irregular_past_participle_adjectives,100.0%,97.7%,90.5%,92.8%,79.4%,99.0%,YES,YES,YES,YES,YES
+irregular_past_participle_verbs,90.0%,86.1%,65.8%,85.3%,79.6%,95.0%,YES,YES,YES,YES,
+irregular_plural_subject_verb_agreement_1,76.7%,92.8%,77.6%,81.1%,72.6%,94.0%,,,,YES,
+irregular_plural_subject_verb_agreement_2,86.7%,92.4%,82.7%,89.1%,87.6%,95.0%,,YES,,,
+left_branch_island_echo_question,20.0%,52.3%,63.0%,31.6%,95.8%,91.0%,,,,,
+left_branch_island_simple_question,13.3%,87.1%,35.5%,36.1%,56.5%,99.0%,,,,,
+matrix_question_npi_licensor_present,10.0%,65.4%,0.7%,1.6%,1.2%,98.0%,,YES,YES,YES,
+npi_present_1,16.7%,64.8%,60.5%,53.6%,46.8%,83.0%,,,,,
+npi_present_2,40.0%,64.3%,48.2%,53.9%,46.9%,96.1%,,,,,
+only_npi_licensor_present,73.3%,94.5%,79.8%,92.9%,56.6%,91.1%,,,,YES,
+only_npi_scope,86.7%,78.5%,45.3%,36.4%,29.8%,72.0%,YES,YES,YES,YES,YES
+passive_1,80.0%,89.3%,80.6%,71.9%,89.2%,95.0%,,,YES,,
+passive_2,83.3%,90.2%,80.9%,72.5%,82.2%,86.0%,,YES,YES,YES,
+principle_A_c_command,76.7%,73.7%,60.8%,58.8%,57.8%,86.0%,YES,YES,YES,YES,
+principle_A_case_1,100.0%,100.0%,100.0%,100.0%,100.0%,98.0%,,,,,YES
+principle_A_case_2,90.0%,94.8%,95.3%,87.3%,48.9%,96.0%,,,YES,YES,
+principle_A_domain_1,100.0%,98.4%,99.1%,98.3%,95.2%,95.0%,YES,YES,YES,YES,YES
+principle_A_domain_2,46.7%,77.5%,69.7%,68.3%,56.4%,75.0%,,,,,
+principle_A_domain_3,56.7%,75.4%,60.0%,55.2%,52.2%,83.0%,,,YES,YES,
+principle_A_reconstruction,46.7%,46.8%,37.9%,46.4%,40.4%,78.2%,,YES,YES,YES,
+regular_plural_subject_verb_agreement_1,80.0%,96.7%,73.1%,89.0%,75.5%,94.1%,,YES,,YES,
+regular_plural_subject_verb_agreement_2,96.7%,91.0%,85.3%,82.5%,80.9%,95.0%,YES,YES,YES,YES,YES
+sentential_negation_npi_licensor_present,100.0%,97.1%,98.8%,100.0%,92.7%,93.1%,YES,YES,,YES,YES
+sentential_negation_npi_scope,43.3%,73.2%,53.3%,23.2%,44.7%,81.0%,,,YES,,
+sentential_subject_island,43.3%,35.5%,36.5%,42.8%,60.5%,60.0%,YES,YES,YES,,
+superlative_quantifiers_1,76.7%,87.0%,84.3%,62.8%,44.7%,91.0%,,,YES,YES,
+superlative_quantifiers_2,66.7%,87.2%,85.2%,82.6%,16.5%,85.0%,,,,YES,
+tough_vs_raising_1,30.0%,72.0%,45.2%,33.6%,32.6%,75.2%,,,,,
+tough_vs_raising_2,73.3%,88.9%,86.1%,93.0%,77.4%,81.0%,,,,,
+transitive,76.7%,86.0%,75.8%,64.7%,70.8%,98.0%,,YES,YES,YES,
+wh_island,70.0%,78.9%,19.5%,46.7%,56.2%,73.3%,,YES,YES,YES,
+wh_questions_object_gap,70.0%,84.2%,60.7%,79.4%,52.7%,85.0%,,YES,,YES,
+wh_questions_subject_gap,93.3%,95.6%,82.5%,91.5%,82.0%,98.0%,,YES,YES,YES,
+wh_questions_subject_gap_long_distance,90.0%,87.1%,86.2%,95.6%,85.6%,85.0%,YES,YES,,YES,YES
+wh_vs_that_no_gap,96.7%,96.8%,86.4%,96.9%,82.5%,97.0%,,YES,,YES,
+wh_vs_that_no_gap_long_distance,96.7%,93.8%,91.2%,97.3%,81.0%,92.0%,YES,YES,,YES,YES
+wh_vs_that_with_gap,36.7%,55.1%,42.0%,42.8%,17.6%,77.0%,,,,YES,
+wh_vs_that_with_gap_long_distance,3.3%,56.3%,17.2%,13.7%,20.0%,74.0%,,,,,
+
+OVERALL AVERAGE,68.3%,83.0%,69.6%,69.8%,61.2%,88.4%
+Gen-3.6 wins (count/67),,13/67,28/67,28/67,43/67,11/67