{
  "scores": [
    {
      "task": "beat",
      "dataset": "GTZAN",
      "metric": "F1",
      "score": 86.27,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "key"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      },
      "evaluation": {
        "num_tracks": 999,
        "matching_tolerance_seconds": 0.07,
        "min_event_time_seconds": 5.0,
        "trim_reference_and_predictions": true,
        "aggregation": "macro_average_per_track_f1"
      }
    },
    {
      "task": "beat",
      "dataset": "osu2017",
      "metric": "F1",
      "score": 93.01,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "key",
          "chord_full"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      },
      "evaluation": {
        "num_tracks": 142,
        "matching_tolerance_seconds": 0.07,
        "min_event_time_seconds": 5.0,
        "trim_reference_and_predictions": true,
        "aggregation": "macro_average_per_track_f1"
      }
    },
    {
      "task": "downbeat",
      "dataset": "GTZAN",
      "metric": "F1",
      "score": 80.45,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "key"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      },
      "evaluation": {
        "num_tracks": 993,
        "matching_tolerance_seconds": 0.07,
        "min_event_time_seconds": 5.0,
        "trim_reference_and_predictions": true,
        "aggregation": "macro_average_per_track_f1"
      }
    },
    {
      "task": "downbeat",
      "dataset": "osu2017",
      "metric": "F1",
      "score": 92.9,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "key",
          "chord_full"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      },
      "evaluation": {
        "num_tracks": 142,
        "matching_tolerance_seconds": 0.07,
        "min_event_time_seconds": 5.0,
        "trim_reference_and_predictions": true,
        "aggregation": "macro_average_per_track_f1"
      }
    },
    {
      "task": "key",
      "dataset": "GiantSteps",
      "metric": "weighted_accuracy",
      "score": 77.73,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "key"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      }
    },
    {
      "task": "key",
      "dataset": "GTZAN",
      "metric": "weighted_accuracy",
      "score": 75.77,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "key"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      }
    },
    {
      "task": "chord",
      "dataset": "osu2017",
      "metric": "majmin",
      "score": 90.08,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "key",
          "chord_full"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      }
    },
    {
      "task": "chord",
      "dataset": "Chords1217",
      "metric": "majmin",
      "score": 83.81,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "chord_full"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      }
    },
    {
      "task": "chord",
      "dataset": "JAAH",
      "metric": "majmin",
      "score": 64.5,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "chord_full"
        ],
        "cuda_version": "12.6"
      },
      "score_unrounded": 64.50302704985715,
      "num_tracks": 113,
      "source": "benchmarks/jaah_results.json",
      "evaluation": {
        "reference": "official_annotations_intersected_with_audio_span",
        "aggregation": "reference_span_weighted_mean_per_track_mir_eval_majmin",
        "eligible_reference_percent": 93.87347199893611,
        "ci95_percent": [
          61.02541843882439,
          67.64783306565447
        ]
      }
    },
    {
      "task": "structure",
      "dataset": "HarmonixSet",
      "metric": "accuracy",
      "score": 80.51,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "structure"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      }
    },
    {
      "task": "structure",
      "dataset": "HarmonixSet",
      "metric": "F1@0.5s",
      "score": 67.96,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "structure"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      }
    },
    {
      "task": "structure",
      "dataset": "HarmonixSet",
      "metric": "F1@3s",
      "score": 82.86,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "structure"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      }
    },
    {
      "task": "melody",
      "dataset": "RWC-Pop",
      "metric": "vocal_pitch_class_F1",
      "score": 82.51,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "structure",
          "key",
          "chord_full",
          "melody_full"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      }
    },
    {
      "task": "melody",
      "dataset": "RWC-Pop",
      "metric": "full_pitch_class_F1",
      "score": 75.29,
      "settings": {
        "preset": "paper",
        "sample_rate": 24000,
        "window_seconds": 300,
        "overlap_seconds": 100,
        "lookahead_seconds": 0,
        "max_length": 5120,
        "decoding": "greedy",
        "attention": "sdpa",
        "weight_dtype": "float32",
        "autocast_dtype": "bfloat16",
        "batch_size": 1,
        "prompts": [
          "timestamp",
          "downbeat_meter",
          "structure",
          "key",
          "chord_full",
          "melody_full"
        ],
        "device": "NVIDIA H800",
        "cuda_version": "12.6"
      }
    }
  ],
  "specialist_baselines": [
    {
      "model": "ChordFormer",
      "task": "chord",
      "dataset": "osu2017",
      "metric": "Maj/min",
      "score": 86.55,
      "score_unrounded": 86.5497453460808,
      "num_tracks": 142,
      "aggregation": "reference_span_weighted_mean_per_track_mir_eval_majmin",
      "inference_protocol": "five_released_models_probability_ensemble",
      "vocabulary": "submission",
      "code_commit": "3da11c078c802b3f5a2ff0f6f8ec184a41537844",
      "source": "benchmarks/chordformer_results.json"
    },
    {
      "model": "ChordFormer",
      "task": "chord",
      "dataset": "Chords1217",
      "metric": "Maj/min",
      "score": 83.94,
      "score_unrounded": 83.94339228883841,
      "num_tracks": 1217,
      "aggregation": "reference_span_weighted_mean_per_track_mir_eval_majmin",
      "inference_protocol": "one_released_held_out_fold_model_per_track",
      "vocabulary": "submission",
      "code_commit": "3da11c078c802b3f5a2ff0f6f8ec184a41537844",
      "source": "benchmarks/chordformer_results.json"
    },
    {
      "model": "ChordFormer",
      "task": "chord",
      "dataset": "JAAH",
      "metric": "Maj/min",
      "score": 59.45,
      "score_unrounded": 59.44669210290719,
      "num_tracks": 113,
      "aggregation": "reference_span_weighted_mean_per_track_mir_eval_majmin",
      "inference_protocol": "five_released_models_probability_ensemble",
      "vocabulary": "submission",
      "code_commit": "3da11c078c802b3f5a2ff0f6f8ec184a41537844",
      "source": "benchmarks/jaah_results.json",
      "ci95_percent": [
        56.02619276510972,
        62.7408426250374
      ]
    }
  ]
}