| { |
| "scores": [ |
| { |
| "task": "beat", |
| "dataset": "GTZAN", |
| "metric": "F1", |
| "score": 86.27, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "key" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| }, |
| "evaluation": { |
| "num_tracks": 999, |
| "matching_tolerance_seconds": 0.07, |
| "min_event_time_seconds": 5.0, |
| "trim_reference_and_predictions": true, |
| "aggregation": "macro_average_per_track_f1" |
| } |
| }, |
| { |
| "task": "beat", |
| "dataset": "osu2017", |
| "metric": "F1", |
| "score": 93.01, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "key", |
| "chord_full" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| }, |
| "evaluation": { |
| "num_tracks": 142, |
| "matching_tolerance_seconds": 0.07, |
| "min_event_time_seconds": 5.0, |
| "trim_reference_and_predictions": true, |
| "aggregation": "macro_average_per_track_f1" |
| } |
| }, |
| { |
| "task": "downbeat", |
| "dataset": "GTZAN", |
| "metric": "F1", |
| "score": 80.45, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "key" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| }, |
| "evaluation": { |
| "num_tracks": 993, |
| "matching_tolerance_seconds": 0.07, |
| "min_event_time_seconds": 5.0, |
| "trim_reference_and_predictions": true, |
| "aggregation": "macro_average_per_track_f1" |
| } |
| }, |
| { |
| "task": "downbeat", |
| "dataset": "osu2017", |
| "metric": "F1", |
| "score": 92.9, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "key", |
| "chord_full" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| }, |
| "evaluation": { |
| "num_tracks": 142, |
| "matching_tolerance_seconds": 0.07, |
| "min_event_time_seconds": 5.0, |
| "trim_reference_and_predictions": true, |
| "aggregation": "macro_average_per_track_f1" |
| } |
| }, |
| { |
| "task": "key", |
| "dataset": "GiantSteps", |
| "metric": "weighted_accuracy", |
| "score": 77.73, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "key" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| } |
| }, |
| { |
| "task": "key", |
| "dataset": "GTZAN", |
| "metric": "weighted_accuracy", |
| "score": 75.77, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "key" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| } |
| }, |
| { |
| "task": "chord", |
| "dataset": "osu2017", |
| "metric": "majmin", |
| "score": 90.08, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "key", |
| "chord_full" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| } |
| }, |
| { |
| "task": "chord", |
| "dataset": "Chords1217", |
| "metric": "majmin", |
| "score": 83.81, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "chord_full" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| } |
| }, |
| { |
| "task": "chord", |
| "dataset": "JAAH", |
| "metric": "majmin", |
| "score": 64.5, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "chord_full" |
| ], |
| "cuda_version": "12.6" |
| }, |
| "score_unrounded": 64.50302704985715, |
| "num_tracks": 113, |
| "source": "benchmarks/jaah_results.json", |
| "evaluation": { |
| "reference": "official_annotations_intersected_with_audio_span", |
| "aggregation": "reference_span_weighted_mean_per_track_mir_eval_majmin", |
| "eligible_reference_percent": 93.87347199893611, |
| "ci95_percent": [ |
| 61.02541843882439, |
| 67.64783306565447 |
| ] |
| } |
| }, |
| { |
| "task": "structure", |
| "dataset": "HarmonixSet", |
| "metric": "accuracy", |
| "score": 80.51, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "structure" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| } |
| }, |
| { |
| "task": "structure", |
| "dataset": "HarmonixSet", |
| "metric": "F1@0.5s", |
| "score": 67.96, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "structure" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| } |
| }, |
| { |
| "task": "structure", |
| "dataset": "HarmonixSet", |
| "metric": "F1@3s", |
| "score": 82.86, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "structure" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| } |
| }, |
| { |
| "task": "melody", |
| "dataset": "RWC-Pop", |
| "metric": "vocal_pitch_class_F1", |
| "score": 82.51, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "structure", |
| "key", |
| "chord_full", |
| "melody_full" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| } |
| }, |
| { |
| "task": "melody", |
| "dataset": "RWC-Pop", |
| "metric": "full_pitch_class_F1", |
| "score": 75.29, |
| "settings": { |
| "preset": "paper", |
| "sample_rate": 24000, |
| "window_seconds": 300, |
| "overlap_seconds": 100, |
| "lookahead_seconds": 0, |
| "max_length": 5120, |
| "decoding": "greedy", |
| "attention": "sdpa", |
| "weight_dtype": "float32", |
| "autocast_dtype": "bfloat16", |
| "batch_size": 1, |
| "prompts": [ |
| "timestamp", |
| "downbeat_meter", |
| "structure", |
| "key", |
| "chord_full", |
| "melody_full" |
| ], |
| "device": "NVIDIA H800", |
| "cuda_version": "12.6" |
| } |
| } |
| ], |
| "specialist_baselines": [ |
| { |
| "model": "ChordFormer", |
| "task": "chord", |
| "dataset": "osu2017", |
| "metric": "Maj/min", |
| "score": 86.55, |
| "score_unrounded": 86.5497453460808, |
| "num_tracks": 142, |
| "aggregation": "reference_span_weighted_mean_per_track_mir_eval_majmin", |
| "inference_protocol": "five_released_models_probability_ensemble", |
| "vocabulary": "submission", |
| "code_commit": "3da11c078c802b3f5a2ff0f6f8ec184a41537844", |
| "source": "benchmarks/chordformer_results.json" |
| }, |
| { |
| "model": "ChordFormer", |
| "task": "chord", |
| "dataset": "Chords1217", |
| "metric": "Maj/min", |
| "score": 83.94, |
| "score_unrounded": 83.94339228883841, |
| "num_tracks": 1217, |
| "aggregation": "reference_span_weighted_mean_per_track_mir_eval_majmin", |
| "inference_protocol": "one_released_held_out_fold_model_per_track", |
| "vocabulary": "submission", |
| "code_commit": "3da11c078c802b3f5a2ff0f6f8ec184a41537844", |
| "source": "benchmarks/chordformer_results.json" |
| }, |
| { |
| "model": "ChordFormer", |
| "task": "chord", |
| "dataset": "JAAH", |
| "metric": "Maj/min", |
| "score": 59.45, |
| "score_unrounded": 59.44669210290719, |
| "num_tracks": 113, |
| "aggregation": "reference_span_weighted_mean_per_track_mir_eval_majmin", |
| "inference_protocol": "five_released_models_probability_ensemble", |
| "vocabulary": "submission", |
| "code_commit": "3da11c078c802b3f5a2ff0f6f8ec184a41537844", |
| "source": "benchmarks/jaah_results.json", |
| "ci95_percent": [ |
| 56.02619276510972, |
| 62.7408426250374 |
| ] |
| } |
| ] |
| } |
|
|
