publish grounding leaderboard

index.html

@@ -112,7 +112,6 @@ python benchmark/score.py --model <your-model> --config en --name "Your Mo
     <strong>Models benchmarked:</strong>
     <a href="https://huggingface.co/nutrientdocs/grounding-en">nutrientdocs/grounding-en</a> ·
     <a href="https://huggingface.co/nutrientdocs/grounding-multilingual">nutrientdocs/grounding-multilingual</a> ·
-    <a href="https://huggingface.co/nutrientdocs/nutrient-document-decision">nutrientdocs/nutrient-document-decision</a> ·
     <a href="https://huggingface.co/cross-encoder/nli-deberta-v3-base">cross-encoder/nli-deberta-v3-base</a> ·
     <a href="https://huggingface.co/cross-encoder/nli-MiniLM2-L6-H768">cross-encoder/nli-MiniLM2-L6-H768</a> ·
     <a href="https://huggingface.co/facebook/bart-large-mnli">facebook/bart-large-mnli</a> ·

leaderboard.json

@@ -267,6 +267,46 @@
     "ctx": 512,
     "ours": false
   },
+  {
+    "name": "OpenAI Decisions (gpt-6-luna)",
+    "model": "gpt-6-luna",
+    "config": "en",
+    "open": false,
+    "n": 5000,
+    "overall_auc": 0.8228269611400488,
+    "auc": {
+      "number": 0.7689031111111111,
+      "date": 0.9890859222932834,
+      "string": 0.9343434343434344,
+      "table": 0.8094126105399382,
+      "prose": 0.9177731738707349
+    },
+    "pairs_per_s": 49.8,
+    "hardware": "OpenAI Decisions API (gpt-6-luna)",
+    "ctx": "api",
+    "cloud": true,
+    "ours": false
+  },
+  {
+    "name": "OpenAI Decisions (gpt-6-luna)",
+    "model": "gpt-6-luna",
+    "config": "multi",
+    "open": false,
+    "n": 8421,
+    "overall_auc": 0.861610311695087,
+    "auc": {
+      "number": 0.9336608888888889,
+      "date": null,
+      "string": null,
+      "table": 0.9352640845070422,
+      "prose": 0.8087951472355769
+    },
+    "pairs_per_s": 51.7,
+    "hardware": "OpenAI Decisions API (gpt-6-luna)",
+    "ctx": "api",
+    "cloud": true,
+    "ours": false
+  },
   {
     "name": "RoBERTa-large MNLI",
     "model": "FacebookAI/roberta-large-mnli",
@@ -323,43 +363,5 @@
     "hardware": "NVIDIA RTX A6000",
     "ctx": 512,
     "ours": false
-  },
-  {
-    "name": "nutrient-document-decision",
-    "model": "nutrientdocs/nutrient-document-decision",
-    "config": "en",
-    "open": false,
-    "n": 5000,
-    "overall_auc": 0.9231032623776494,
-    "auc": {
-      "number": 0.9413457777777777,
-      "date": 0.999257438256781,
-      "string": 0.9968885281385281,
-      "table": 0.9200564106251071,
-      "prose": 0.9593182032206422
-    },
-    "pairs_per_s": 8.2,
-    "hardware": "NVIDIA RTX A6000",
-    "ctx": 8192,
-    "ours": true
-  },
-  {
-    "name": "nutrient-document-decision",
-    "model": "nutrientdocs/nutrient-document-decision",
-    "config": "multi",
-    "open": false,
-    "n": 8421,
-    "overall_auc": 0.948770067424967,
-    "auc": {
-      "number": 0.9969035555555555,
-      "date": null,
-      "string": null,
-      "table": 0.9954755455811793,
-      "prose": 0.8926265024038461
-    },
-    "pairs_per_s": 7.8,
-    "hardware": "NVIDIA RTX A6000",
-    "ctx": 8192,
-    "ours": true
   }
 ]

results/openai-decisions-gpt-6-luna-en.json

@@ -0,0 +1,26 @@
+{
+  "name": "OpenAI Decisions (gpt-6-luna)",
+  "model": "gpt-6-luna",
+  "config": "en",
+  "open": false,
+  "n": 5000,
+  "overall_auc": 0.8228269611400488,
+  "auc": {
+    "number": 0.7689031111111111,
+    "date": 0.9890859222932834,
+    "string": 0.9343434343434344,
+    "table": 0.8094126105399382,
+    "prose": 0.9177731738707349
+  },
+  "pairs_per_s": 49.8,
+  "hardware": "OpenAI Decisions API (gpt-6-luna)",
+  "ctx": "api",
+  "perf": {
+    "accelerator": "OpenAI Decisions API (gpt-6-luna)",
+    "n_pairs": 5000,
+    "elapsed_s": 100.36,
+    "ctx": "api",
+    "pairs_per_s": 49.8
+  },
+  "cloud": true
+}

results/openai-decisions-gpt-6-luna-multi.json

@@ -0,0 +1,26 @@
+{
+  "name": "OpenAI Decisions (gpt-6-luna)",
+  "model": "gpt-6-luna",
+  "config": "multi",
+  "open": false,
+  "n": 8421,
+  "overall_auc": 0.861610311695087,
+  "auc": {
+    "number": 0.9336608888888889,
+    "date": null,
+    "string": null,
+    "table": 0.9352640845070422,
+    "prose": 0.8087951472355769
+  },
+  "pairs_per_s": 51.7,
+  "hardware": "OpenAI Decisions API (gpt-6-luna)",
+  "ctx": "api",
+  "perf": {
+    "accelerator": "OpenAI Decisions API (gpt-6-luna)",
+    "n_pairs": 8421,
+    "elapsed_s": 162.83,
+    "ctx": "api",
+    "pairs_per_s": 51.7
+  },
+  "cloud": true
+}