Upload from GitHub Actions: guard main.py against partial-scale HF pushes; restore aggregated results

A smoke run earlier today (N_LANGUAGES=5 N_MODELS=150) silently
overwrote fair-forward/evals-for-every-language-results with a
filtered subset: the aggregation step at main.py:97 filters
results_agg to current_models x current_languages, then save()
pushes regardless of scale, so a 5-language smoke truncated the
dashboard from ~1000 langs to 5. results-detailed was untouched
(it merges full history), which made recovery possible.

Restoration: re-aggregated the entire results-detailed table
(701k rows, 135 models, 221 langs -> 70.9k aggregate rows) and
pushed only the `results` dataset back. Local results/*.json
snapshots now reflect the restored state.

Footgun patch:
- evals/main.py:
* Add CANONICAL_N_LANGUAGES_FOR_PUSH (1000) and
CANONICAL_N_MODELS_FOR_PUSH (100) constants.
* Compute ALLOW_HF_PUSH_RESULTS at module load. When either env
scale is below canonical, route results/models/languages
through save_local_only() instead of save(), keeping the
public dataset intact.
* results-detailed is always pushed (immutable log, safe to
append from any scale).
- evals/datasets_/util.py: add save_local_only() helper that
writes the local snapshot without touching HF.

Now: smoke tests can run with reduced scale without risk to the
published view.

.github/workflows/nightly-evals.yml

@@ -1,8 +1,8 @@
 name: Nightly Evaluation Run
 
 on:
-  # schedule:
-  #   - cron: '0 3 * * *'  # Run at 3am UTC every day
+  schedule:
+    - cron: '0 3 * * 1'  # Weekly: Mondays 3am UTC
   workflow_dispatch:  # Allow manual triggering
 
 jobs:
@@ -31,7 +31,9 @@ jobs:
           N_SENTENCES: 10
           # Keep these aligned with defaults in evals/main.py for comparability
           N_LANGUAGES: 1000
-          N_MODELS: 40
+          # Bumped 2026-05-19 from 40 to 150 to cover the auto-discovered cohort
+          # (typically ~100 models after dedupe + cost cap).
+          N_MODELS: 150
         run: |
           uv run huggingface-cli login --token ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
           uv run evals/download_data.py

evals/datasets_/util.py

@@ -81,6 +81,18 @@ def save(df: pd.DataFrame, fname: str):
     df.to_json(f"results/{fname}.json", orient="records", force_ascii=False, indent=2)
 
 
+def save_local_only(df: pd.DataFrame, fname: str):
+    """Write the snapshot to results/{fname}.json without pushing to HF.
+
+    Used during partial-scale eval runs (smoke tests, local development) so
+    the public dataset isn't truncated by a filtered aggregate. The next
+    full-scale run will push the canonical version.
+    """
+    df = df.drop(columns=["__index_level_0__"], errors="ignore")
+    Path("results").mkdir(exist_ok=True)
+    df.to_json(f"results/{fname}.json", orient="records", force_ascii=False, indent=2)
+
+
 def get_valid_task_languages(task_name: str) -> set:
     """Return set of bcp_47 codes that have data available for the given task."""
     from datasets_.flores import flores, splits

evals/main.py

@@ -9,13 +9,28 @@ from models import models
 from rich import print
 from tasks import tasks
 from tqdm.asyncio import tqdm_asyncio
-from datasets_.util import load, save, get_valid_task_languages
+from datasets_.util import load, save, save_local_only, get_valid_task_languages
 from tqdm import tqdm
 
+# Canonical scale used in the nightly workflow. Reduced scale (smaller
+# N_LANGUAGES or N_MODELS) is OK for local validation, but pushing the
+# aggregated `results` dataset back to HF in that mode would truncate the
+# published table — see CANONICAL_*_FOR_PUSH and the guard in save() below.
+CANONICAL_N_LANGUAGES_FOR_PUSH = 1000
+CANONICAL_N_MODELS_FOR_PUSH = 100  # nightly uses 150; bar is "covers the full cohort"
+
 n_sentences = int(environ.get("N_SENTENCES", 10))
 n_languages = int(environ.get("N_LANGUAGES", 1000))
 n_models = int(environ.get("N_MODELS", 40))
 
+# When n_languages or n_models is smaller than canonical, the filter in
+# `results_agg` below would discard most rows and overwrite the public HF
+# aggregate. Detect that and downgrade to local-only writes.
+ALLOW_HF_PUSH_RESULTS = (
+    n_languages >= CANONICAL_N_LANGUAGES_FOR_PUSH
+    and n_models >= CANONICAL_N_MODELS_FOR_PUSH
+)
+
 async def evaluate():
     start_time = time.time()
 
@@ -86,10 +101,29 @@ async def evaluate():
         .reset_index()
     )
     
+    # results-detailed is append-merged (immutable log); safe to push from any scale.
     save(all_results, "results-detailed")
-    save(results_agg, "results")
-    save(models, "models")
-    save(languages, "languages")
+
+    # The aggregated tables are filtered by current_models × current_languages,
+    # so a partial-scale run (small N_LANGUAGES / N_MODELS) would truncate the
+    # published view. Refuse to push in that case; write locally only.
+    if ALLOW_HF_PUSH_RESULTS:
+        save(results_agg, "results")
+        save(models, "models")
+        save(languages, "languages")
+    else:
+        print(
+            f"[main] partial-scale run "
+            f"(N_LANGUAGES={n_languages}, N_MODELS={n_models}); "
+            f"writing aggregates LOCALLY only (skip HF push) to protect the "
+            f"public dataset. Push from a full-scale run "
+            f"(>={CANONICAL_N_LANGUAGES_FOR_PUSH} langs, "
+            f">={CANONICAL_N_MODELS_FOR_PUSH} models)."
+        )
+        save_local_only(results_agg, "results")
+        save_local_only(models, "models")
+        save_local_only(languages, "languages")
+
     elapsed = time.time() - start_time
     print(f"Evaluation completed in {str(timedelta(seconds=int(elapsed)))}")
 

evals/models.py

@@ -1,6 +1,7 @@
 import re
 from datetime import date
 from os import getenv
+from pathlib import Path
 
 import pandas as pd
 from aiolimiter import AsyncLimiter
@@ -63,6 +64,14 @@ important_models = [
     "amazon/nova-pro-v1",  # 0.09$
     "moonshotai/kimi-k2",  # 0.6$
     "baidu/ernie-4.5-300b-a47b",
+    # Added 2026-05-19 — new-generation flagships (one per family; auto-discovery handles the rest)
+    "openai/gpt-5.5",  # $30/M output; gpt-5.5-pro is $180/M, beyond cap
+    "anthropic/claude-opus-4.7",
+    "deepseek/deepseek-v4-pro",
+    "x-ai/grok-4.20",
+    "mistralai/mistral-medium-3.5",
+    "moonshotai/kimi-k2.6",
+    "google/gemini-3.1-flash-lite",
 ]
 
 blocklist = [
@@ -84,6 +93,11 @@ blocklist = [
     "qwen/qwen3-235b-a22b",  # ~60% ok, content=None often
 ]
 
+# Hard upper bound on per-token output cost. Models above this are dropped
+# (validated in get_or_metadata + discover_new_models + load_models filter).
+# Raised 2026-05-19 from $25 -> $30 to accommodate GPT-5.5 ($30/M output).
+COST_CAP_PER_1M = 30.0
+
 transcription_models = [
     "elevenlabs/scribe_v1",
     "openai/whisper-large-v3",
@@ -116,80 +130,137 @@ def get_or_metadata(permaslug):
     return slugs[0] if len(slugs) >= 1 else None
 
 
-@cache
-def get_historical_popular_models(date: date):
-    # date parameter is used for daily caching
-    try:
-        raw = get("https://openrouter.ai/rankings").text
-
-        # Extract model data from rankingData using regex
-        # Find all count and model_permaslug pairs in the data
-        # Format: "count":number,"model_permaslug":"model/name"
-        pattern = r"\\\"count\\\":([\d.]+).*?\\\"model_permaslug\\\":\\\"([^\\\"]+)\\\""
-        matches = re.findall(pattern, raw)
-
-        if matches:
-            # Aggregate model counts
-            model_counts = {}
-            for count_str, model_slug in matches:
-                count = float(count_str)
-                if not model_slug.startswith("openrouter") and model_slug != "Others":
-                    # Remove variant suffixes for aggregation
-                    base_model = model_slug.split(":")[0]
-                    model_counts[base_model] = model_counts.get(base_model, 0) + count
-
-            # Sort by popularity and return top models
-            sorted_models = sorted(
-                model_counts.items(), key=lambda x: x[1], reverse=True
-            )
-            result = []
-            for model_slug, count in sorted_models:
-                result.append({"slug": model_slug, "count": int(count)})
-
-            return result
-        else:
-            return []
+# Strip numeric version tokens AND date-snapshot suffixes from a slug to
+# derive a model "family" key. Size-tier suffixes (-pro, -mini, -flash,
+# -lite, -opus, -haiku, ...) stay so flagship and cheap-tier variants form
+# separate families.
+#
+# Examples:
+#   openai/gpt-5.5-pro                -> openai/gpt-pro
+#   openai/gpt-5.4-mini               -> openai/gpt-mini
+#   anthropic/claude-opus-4.7         -> anthropic/claude-opus
+#   deepseek/deepseek-v4-pro          -> deepseek/deepseek-pro
+#   meta-llama/llama-3.3-70b-instruct -> meta-llama/llama-70b-instruct
+#   qwen/qwen3-235b-a22b-07-25        -> qwen/qwen-235b-a22b  (date suffix stripped)
+#   bytedance-seed/seed-1.6-20250625  -> bytedance-seed/seed
+_DATE_SUFFIX_RE = re.compile(
+    r"-(20\d{6}|20\d{2}-\d{2}-\d{2}|\d{2}-\d{2}|\d{4})$"
+)
+_VERSION_SUFFIX_RE = re.compile(r"[-]?v?\d+(\.\d+)*(-exp|-instruct)?(?=($|-))")
+
+
+def _family_key(slug: str) -> str:
+    # Strip trailing date snapshots first (so version regex can match cleanly).
+    while True:
+        new = _DATE_SUFFIX_RE.sub("", slug)
+        if new == slug:
+            break
+        slug = new
+    return _VERSION_SUFFIX_RE.sub("", slug)
+
+
+# Providers we trust to ship general-purpose text LLMs. Adding a new vendor
+# here is the explicit human gate for auto-discovery.
+_DISCOVERY_PROVIDER_ALLOWLIST = frozenset({
+    "openai", "anthropic", "google", "meta-llama", "mistralai", "deepseek",
+    "x-ai", "qwen", "alibaba", "cohere", "amazon", "moonshotai", "baidu",
+    "allenai", "microsoft", "liquid", "ibm-granite", "nvidia", "rekaai",
+    "stepfun", "tencent", "z-ai", "bytedance-seed", "ai21", "nousresearch",
+    "perplexity", "arcee-ai", "deepcogito", "prime-intellect", "writer",
+    "upstage", "openrouter",
+})
+
+# Skip these substrings anywhere in the slug — covers transient snapshots,
+# non-text modalities, and task-specialised variants.
+_DISCOVERY_SKIP_TAGS = (
+    "-preview", "-beta", "-experimental", ":free", "-latest",
+    "-vision", "-vl", "-image", "-audio", "-tts", "-stt", "-embed",
+    "-asr", "-transcribe", "-search", "rerank", "-ocr", "-edit",
+    "coder", "codex", "devstral", "codestral",
+    "-thinking", "-reasoning", "-think", "-deep-research", "deepresearch",
+    "-multi-agent", "safeguard",
+)
 
-    except Exception as e:
-        return []
+# Skip these whole product families (named non-text models).
+_DISCOVERY_SKIP_PRODUCTS = (
+    "whisper", "voxtral", "chirp", "kokoro", "orpheus", "zonos", "csm-",
+    "sora", "veo-", "wan-", "seedance", "seedream", "flux.", "imagine",
+    "kling", "hailuo", "riverflow", "recraft", "morph-",
+    "bge-", "gte-", "e5-", "multilingual-e5",
+)
 
 
 @cache
-def get_current_popular_models(date: date):
-    # date parameter is used for daily caching
-    try:
-        raw = get("https://openrouter.ai/rankings?view=day").text
-
-        # Extract model data from daily rankings
-        # Find all count and model_permaslug pairs in the daily data
-        pattern = r"\\\"count\\\":([\d.]+).*?\\\"model_permaslug\\\":\\\"([^\\\"]+)\\\""
-        matches = re.findall(pattern, raw)
-
-        if matches:
-            # Aggregate model counts
-            model_counts = {}
-            for count_str, model_slug in matches:
-                count = float(count_str)
-                if not model_slug.startswith("openrouter") and model_slug != "Others":
-                    # Remove variant suffixes for aggregation
-                    base_model = model_slug.split(":")[0]
-                    model_counts[base_model] = model_counts.get(base_model, 0) + count
-
-            # Sort by popularity and return top models
-            sorted_models = sorted(
-                model_counts.items(), key=lambda x: x[1], reverse=True
-            )
-            result = []
-            for model_slug, count in sorted_models:
-                result.append({"slug": model_slug, "count": int(count)})
-
-            return result
-        else:
-            return []
+def discover_new_models(date: date) -> list[str]:
+    """Surface OpenRouter models matching inclusion rules; pick the flagship per family.
 
+    Flagship = highest-cost non-blocked variant within a family. If a model's
+    flagship gets auto-blocklisted, the next-most-expensive variant takes its
+    place on the next call (auto_blocklist is consulted before the dedupe step).
+    """
+    try:
+        catalog = load_or_metadata(date)
     except Exception as e:
+        print(f"[discover_new_models] OpenRouter catalog fetch failed: {e}; skipping")
         return []
 
+    cutoff = pd.Timestamp.now(tz="UTC") - pd.Timedelta(days=365)
+    curated_families = {_family_key(s) for s in important_models}
+    blocked_families = {_family_key(s) for s in blocklist}
+    blocked = set(blocklist) | set(load_auto_blocklist(date))
+
+    candidates = []
+    for m in catalog:
+        slug = m.get("permaslug") or m.get("slug")
+        if not slug or slug in blocked:
+            continue
+        if slug.startswith("~"):
+            continue  # OpenRouter alias slugs like "~anthropic/claude-opus-latest"
+        provider = slug.split("/", 1)[0] if "/" in slug else ""
+        if provider not in _DISCOVERY_PROVIDER_ALLOWLIST:
+            continue
+        if any(tag in slug for tag in _DISCOVERY_SKIP_TAGS):
+            continue
+        slug_lower = slug.lower()
+        if any(prod in slug_lower for prod in _DISCOVERY_SKIP_PRODUCTS):
+            continue
+        if not m.get("endpoint"):
+            continue
+        if m["endpoint"].get("is_free"):
+            continue
+        try:
+            trains = m["endpoint"]["provider_info"]["dataPolicy"]["training"]
+        except (TypeError, KeyError):
+            continue
+        if trains is not False:
+            continue
+        try:
+            cost_per_1m = float(m["endpoint"]["pricing"]["completion"]) * 1_000_000
+        except (TypeError, KeyError, ValueError):
+            continue
+        if cost_per_1m > COST_CAP_PER_1M:
+            continue
+        try:
+            created = pd.to_datetime(m["created_at"], utc=True)
+        except (TypeError, ValueError, KeyError):
+            continue
+        if created < cutoff:
+            continue
+        family = _family_key(slug)
+        if family in curated_families:
+            continue  # already represented in important_models — don't duplicate
+        if family in blocked_families:
+            continue  # date-suffixed snapshot of a blocklisted slug
+        candidates.append((slug, created, family, cost_per_1m))
+
+    # Dedupe: pick flagship per family (highest cost wins; newer wins on ties).
+    by_family: dict[str, tuple[str, tuple]] = {}
+    for slug, created, family, cost in candidates:
+        rank = (-cost, -created.timestamp())
+        if family not in by_family or rank < by_family[family][1]:
+            by_family[family] = (slug, rank)
+    return sorted(s for s, _ in by_family.values())
+
 
 def get_translation_models():
     return pd.DataFrame(
@@ -336,15 +407,91 @@ def get_training_policy(row):
     return row["endpoint"]["provider_info"]["dataPolicy"]["training"]
 
 
+# Auto-blocklist thresholds: a model is auto-excluded if it has attempted at
+# least MIN_ATTEMPTS evaluations and FAIL_PCT_THRESHOLD% or more returned an
+# error (content=None / filtered / etc.). Matches the manual blocklist's
+# existing quality bar ("~33% ok, content=None often" -> 67% fail -> blocked).
+AUTO_BLOCKLIST_MIN_ATTEMPTS = 100
+AUTO_BLOCKLIST_FAIL_PCT_THRESHOLD = 50.0
+
+
+def compute_model_health() -> pd.DataFrame:
+    """Per-model success/failure stats from results-detailed. Empty DF on miss."""
+    from datasets_.util import load
+
+    detailed = load("results-detailed")
+    if detailed.empty or "status" not in detailed.columns:
+        return pd.DataFrame(
+            columns=["model", "total", "failed", "failed_pct", "score_nonfailed"]
+        )
+    is_error = (detailed["status"] != "ok").rename("is_error")
+    grouped = pd.DataFrame(
+        {
+            "total": detailed.groupby("model").size(),
+            "failed": is_error.groupby(detailed["model"]).sum(),
+            "score_nonfailed": detailed[~is_error].groupby("model")["score"].mean(),
+        }
+    ).reset_index()
+    grouped["failed_pct"] = grouped["failed"] / grouped["total"] * 100
+    return grouped.sort_values("failed_pct", ascending=False)
+
+
+@cache
+def load_auto_blocklist(date: date) -> list[str]:
+    """Models past the failure threshold in observed history. Empty on first run."""
+    try:
+        health = compute_model_health()
+    except Exception as e:
+        print(f"[auto_blocklist] failed to load history: {e}; using empty list")
+        return []
+    if health.empty:
+        return []
+    bad = health[
+        (health["total"] >= AUTO_BLOCKLIST_MIN_ATTEMPTS)
+        & (health["failed_pct"] >= AUTO_BLOCKLIST_FAIL_PCT_THRESHOLD)
+    ]
+    return sorted(bad["model"].tolist())
+
+
 @cache
 def load_models(date: date) -> pd.DataFrame:
-    # popular_models = (
-    #     get_historical_popular_models(date.today())[:20]
-    #     + get_current_popular_models(date.today())[:10]
-    # )
-    popular_models = []
-    popular_models = [m["slug"] for m in popular_models]
-    all_model_candidates = set(important_models + popular_models) - set(blocklist)
+    auto_discovered = discover_new_models(date)
+    auto_blocked = set(load_auto_blocklist(date))
+
+    # Manual curation wins: important_models override the auto-blocklist
+    # (the warning gives a human a nudge to investigate the quality regression).
+    override = set(important_models) & auto_blocked
+    if override:
+        print(
+            f"[load_models] important_models override auto_blocklist (kept anyway): "
+            f"{sorted(override)}"
+        )
+    if auto_blocked - override:
+        print(
+            f"[load_models] auto_blocklist excluding: "
+            f"{sorted(auto_blocked - override)}"
+        )
+    if auto_discovered:
+        print(f"[load_models] auto_discovered added: {auto_discovered}")
+
+    all_model_candidates = (
+        (set(important_models) | (set(auto_discovered) - auto_blocked))
+        - set(blocklist)
+    )
+
+    # Snapshot health stats for inspection (small enough to track in git).
+    try:
+        health = compute_model_health()
+        if not health.empty:
+            Path("results").mkdir(exist_ok=True)
+            health.to_json(
+                "results/model_health.json",
+                orient="records",
+                indent=2,
+                force_ascii=False,
+            )
+    except Exception as e:
+        print(f"[load_models] failed to snapshot model_health.json: {e}")
 
     # Validate models exist on OpenRouter before including them
     valid_models = []
@@ -380,8 +527,16 @@ def load_models(date: date) -> pd.DataFrame:
     models.to_json(
         "models_unfiltered.json", orient="records", indent=2, force_ascii=False
     )
-    # Filter out expensive models to keep costs reasonable
-    models = models[models["cost"] <= 25.0].reset_index(drop=True)
+    # Filter out expensive models to keep costs reasonable.
+    # Log any manually-curated entries that get dropped here so the user knows why.
+    too_expensive = models[models["cost"] > COST_CAP_PER_1M]
+    important_dropped = too_expensive[too_expensive["id"].isin(important_models)]
+    for _, row in important_dropped.iterrows():
+        print(
+            f"[load_models] dropping {row['id']} from cohort: "
+            f"cost ${row['cost']}/M > cap ${COST_CAP_PER_1M}/M"
+        )
+    models = models[models["cost"] <= COST_CAP_PER_1M].reset_index(drop=True)
     models["tasks"] = [
         [
             "translation_from",

notes/system-architecture-diagram.md

@@ -18,7 +18,7 @@ flowchart TD
     H --> |Save| I[models.json]
     
     %% Model Validation & Cost Filtering
-    H --> |"Validate Models<br/>Check API Availability<br/>No User Data Training"| H1["Valid Models Only<br/>Cost ≤ $25/1M tokens"]
+    H --> |"Validate Models<br/>Check API Availability<br/>No User Data Training"| H1["Valid Models Only<br/>Cost ≤ $30/1M tokens"]
     H1 --> H2["Robust Model List<br/>Default: Top 40 models"]
     
     %% Language Data
@@ -131,7 +131,7 @@ flowchart TD
 - **Static Curated Models**: Handpicked important models (~42 models) for comprehensive evaluation
 - **Dynamic Popular Models**: Web scraping capability available but currently disabled
 - **Quality Control**: Blocklist for problematic or incompatible models
-- **Model Validation**: API availability checks, cost filtering (≤$25/1M tokens), and inclusion only when OpenRouter metadata shows at least one privacy-compatible provider
+- **Model Validation**: API availability checks, cost filtering (≤$30/1M tokens), and inclusion only when OpenRouter metadata shows at least one privacy-compatible provider
 - **Default Selection**: Top 40 models by default (configurable via N_MODELS)
 - **Metadata Enrichment**: Rich model information from OpenRouter and HuggingFace APIs
 
@@ -179,7 +179,7 @@ flowchart TD
 
 ## Data Flow Summary
 
-1. **Model Discovery**: Load curated models (~42) → validate API availability and cost (≤$25/1M tokens) → exclude providers training on user data → enrich with metadata from OpenRouter and HuggingFace
+1. **Model Discovery**: Load curated models (~42) → validate API availability and cost (≤$30/1M tokens) → exclude providers training on user data → enrich with metadata from OpenRouter and HuggingFace
 2. **Evaluation Setup**: Generate all valid Model × Language × Task combinations (default: 40 models × 1000 languages) with pre-computed language filtering and origin tracking
 3. **Task Execution**: Run evaluations using unified English prompting with reasoning templates, batch processing (2000 per batch), and rate limiting
 4. **Result Processing**: Aggregate scores by model+language+task+origin, compute bootstrap confidence intervals, and save to JSON files (results.json and results-detailed.json)

results/languages.json

@@ -7,7 +7,7 @@
     "family":"Indo-European",
     "flores_path":"eng_Latn",
     "fleurs_tag":"en_us",
-    "commonvoice_hours":2714.0,
+    "commonvoice_hours":2739.0,
     "commonvoice_locale":"en",
     "in_benchmark":true
   },
@@ -19,7 +19,7 @@
     "family":"Sino-Tibetan",
     "flores_path":"cmn_Hans",
     "fleurs_tag":"cmn_hans_cn",
-    "commonvoice_hours":425.0,
+    "commonvoice_hours":426.0,
     "commonvoice_locale":"zh-TW",
     "in_benchmark":true
   },
@@ -43,7 +43,7 @@
     "family":"Indo-European",
     "flores_path":"spa_Latn",
     "fleurs_tag":"es_419",
-    "commonvoice_hours":451.0,
+    "commonvoice_hours":453.0,
     "commonvoice_locale":"es",
     "in_benchmark":true
   },
@@ -79,7 +79,7 @@
     "family":"Indo-European",
     "flores_path":"fra_Latn",
     "fleurs_tag":"fr_fr",
-    "commonvoice_hours":1097.0,
+    "commonvoice_hours":1103.0,
     "commonvoice_locale":"fr",
     "in_benchmark":true
   },
@@ -103,7 +103,7 @@
     "family":"Indo-European",
     "flores_path":"por_Latn",
     "fleurs_tag":"pt_br",
-    "commonvoice_hours":182.0,
+    "commonvoice_hours":183.0,
     "commonvoice_locale":"pt",
     "in_benchmark":true
   },
@@ -127,7 +127,7 @@
     "family":"Indo-European",
     "flores_path":"rus_Cyrl",
     "fleurs_tag":"ru_ru",
-    "commonvoice_hours":250.0,
+    "commonvoice_hours":252.0,
     "commonvoice_locale":"ru",
     "in_benchmark":true
   },
@@ -163,7 +163,7 @@
     "family":"Indo-European",
     "flores_path":"deu_Latn",
     "fleurs_tag":"de_de",
-    "commonvoice_hours":1390.0,
+    "commonvoice_hours":1394.0,
     "commonvoice_locale":"de",
     "in_benchmark":true
   },
@@ -235,7 +235,7 @@
     "family":"Austroasiatic",
     "flores_path":"vie_Latn",
     "fleurs_tag":"vi_vn",
-    "commonvoice_hours":7.1,
+    "commonvoice_hours":7.6,
     "commonvoice_locale":"vi",
     "in_benchmark":true
   },
@@ -259,7 +259,7 @@
     "family":"Indo-European",
     "flores_path":"pes_Arab",
     "fleurs_tag":"fa_ir",
-    "commonvoice_hours":372.0,
+    "commonvoice_hours":373.0,
     "commonvoice_locale":"fa",
     "in_benchmark":true
   },
@@ -319,7 +319,7 @@
     "family":"Indo-European",
     "flores_path":"ita_Latn",
     "fleurs_tag":"it_it",
-    "commonvoice_hours":364.0,
+    "commonvoice_hours":365.0,
     "commonvoice_locale":"it",
     "in_benchmark":true
   },
@@ -379,7 +379,7 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":"ps_af",
-    "commonvoice_hours":3087.0,
+    "commonvoice_hours":3210.0,
     "commonvoice_locale":"ps",
     "in_benchmark":false
   },
@@ -547,7 +547,7 @@
     "family":"Afro-Asiatic",
     "flores_path":"gaz_Latn",
     "fleurs_tag":"om_et",
-    "commonvoice_hours":0.0,
+    "commonvoice_hours":24.0,
     "commonvoice_locale":"om",
     "in_benchmark":true
   },
@@ -643,7 +643,7 @@
     "family":"Indo-European",
     "flores_path":"ukr_Cyrl",
     "fleurs_tag":"uk_ua",
-    "commonvoice_hours":100.0,
+    "commonvoice_hours":101.0,
     "commonvoice_locale":"uk",
     "in_benchmark":true
   },
@@ -655,7 +655,7 @@
     "family":"Atlantic-Congo",
     "flores_path":"yor_Latn",
     "fleurs_tag":"yo_ng",
-    "commonvoice_hours":6.4,
+    "commonvoice_hours":6.5,
     "commonvoice_locale":"yo",
     "in_benchmark":true
   },
@@ -679,7 +679,7 @@
     "family":"Atlantic-Congo",
     "flores_path":"ibo_Latn",
     "fleurs_tag":"ig_ng",
-    "commonvoice_hours":2.3,
+    "commonvoice_hours":2.5,
     "commonvoice_locale":"ig",
     "in_benchmark":true
   },
@@ -751,7 +751,7 @@
     "family":"Indo-European",
     "flores_path":"ron_Latn",
     "fleurs_tag":"ro_ro",
-    "commonvoice_hours":21.0,
+    "commonvoice_hours":24.0,
     "commonvoice_locale":"ro",
     "in_benchmark":true
   },
@@ -775,7 +775,7 @@
     "family":"Indo-European",
     "flores_path":"npi_Deva",
     "fleurs_tag":"ne_np",
-    "commonvoice_hours":1.3,
+    "commonvoice_hours":1.4,
     "commonvoice_locale":"ne-NP",
     "in_benchmark":true
   },
@@ -931,7 +931,7 @@
     "family":"Austroasiatic",
     "flores_path":"khm_Khmr",
     "fleurs_tag":"km_kh",
-    "commonvoice_hours":0.0,
+    "commonvoice_hours":0.1,
     "commonvoice_locale":"km",
     "in_benchmark":true
   },
@@ -1027,7 +1027,7 @@
     "family":"Uralic",
     "flores_path":"hun_Latn",
     "fleurs_tag":"hu_hu",
-    "commonvoice_hours":129.0,
+    "commonvoice_hours":135.0,
     "commonvoice_locale":"hu",
     "in_benchmark":true
   },
@@ -1075,7 +1075,7 @@
     "family":"Atlantic-Congo",
     "flores_path":"twi_Latn_akua1239",
     "fleurs_tag":null,
-    "commonvoice_hours":0.2,
+    "commonvoice_hours":0.3,
     "commonvoice_locale":"tw",
     "in_benchmark":true
   },
@@ -1195,7 +1195,7 @@
     "family":"Indo-European",
     "flores_path":"bel_Cyrl",
     "fleurs_tag":"be_by",
-    "commonvoice_hours":1819.0,
+    "commonvoice_hours":1821.0,
     "commonvoice_locale":"be",
     "in_benchmark":true
   },
@@ -1303,7 +1303,7 @@
     "family":"Indo-European",
     "flores_path":"cat_Latn",
     "fleurs_tag":"ca_es",
-    "commonvoice_hours":2945.0,
+    "commonvoice_hours":2976.0,
     "commonvoice_locale":"ca",
     "in_benchmark":true
   },
@@ -1387,7 +1387,7 @@
     "family":"Turkic",
     "flores_path":"uig_Arab",
     "fleurs_tag":null,
-    "commonvoice_hours":453.0,
+    "commonvoice_hours":460.0,
     "commonvoice_locale":"ug",
     "in_benchmark":true
   },
@@ -1411,7 +1411,7 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":0.7,
+    "commonvoice_hours":0.8,
     "commonvoice_locale":"gsw",
     "in_benchmark":false
   },
@@ -1435,7 +1435,7 @@
     "family":"Afro-Asiatic",
     "flores_path":"zgh_Tfng",
     "fleurs_tag":null,
-    "commonvoice_hours":1.4,
+    "commonvoice_hours":1.5,
     "commonvoice_locale":"zgh",
     "in_benchmark":true
   },
@@ -1555,7 +1555,7 @@
     "family":"Indo-European",
     "flores_path":"als_Latn",
     "fleurs_tag":null,
-    "commonvoice_hours":9.0,
+    "commonvoice_hours":9.1,
     "commonvoice_locale":"sq",
     "in_benchmark":true
   },
@@ -1711,7 +1711,7 @@
     "family":"Atlantic-Congo",
     "flores_path":"lug_Latn",
     "fleurs_tag":"lg_ug",
-    "commonvoice_hours":437.0,
+    "commonvoice_hours":438.0,
     "commonvoice_locale":"lg",
     "in_benchmark":true
   },
@@ -1771,7 +1771,7 @@
     "family":"Indo-European",
     "flores_path":"hye_Armn",
     "fleurs_tag":"hy_am",
-    "commonvoice_hours":34.0,
+    "commonvoice_hours":37.0,
     "commonvoice_locale":"hy-AM",
     "in_benchmark":true
   },
@@ -2143,7 +2143,7 @@
     "family":"Kartvelian",
     "flores_path":"kat_Geor",
     "fleurs_tag":"ka_ge",
-    "commonvoice_hours":168.0,
+    "commonvoice_hours":241.0,
     "commonvoice_locale":"ka",
     "in_benchmark":true
   },
@@ -2155,7 +2155,7 @@
     "family":"Indo-European",
     "flores_path":"glg_Latn",
     "fleurs_tag":"gl_es",
-    "commonvoice_hours":290.0,
+    "commonvoice_hours":311.0,
     "commonvoice_locale":"gl",
     "in_benchmark":true
   },
@@ -2215,8 +2215,8 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":0.0,
+    "commonvoice_locale":"tiv",
     "in_benchmark":false
   },
   {
@@ -2227,7 +2227,7 @@
     "family":"Afro-Asiatic",
     "flores_path":"kab_Latn",
     "fleurs_tag":null,
-    "commonvoice_hours":571.0,
+    "commonvoice_hours":572.0,
     "commonvoice_locale":"kab",
     "in_benchmark":true
   },
@@ -2371,8 +2371,8 @@
     "family":"Atlantic-Congo",
     "flores_path":"sag_Latn",
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":0.0,
+    "commonvoice_locale":"sg",
     "in_benchmark":true
   },
   {
@@ -2503,7 +2503,7 @@
     "family":"Indo-European",
     "flores_path":"lit_Latn",
     "fleurs_tag":"lt_lt",
-    "commonvoice_hours":26.0,
+    "commonvoice_hours":27.0,
     "commonvoice_locale":"lt",
     "in_benchmark":true
   },
@@ -3079,8 +3079,8 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":0.0,
+    "commonvoice_locale":"bin",
     "in_benchmark":false
   },
   {
@@ -3463,7 +3463,7 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":1.9,
+    "commonvoice_hours":2.0,
     "commonvoice_locale":"zza",
     "in_benchmark":false
   },
@@ -3475,7 +3475,7 @@
     "family":"Indo-European",
     "flores_path":"lvs_Latn",
     "fleurs_tag":"lv_lv",
-    "commonvoice_hours":266.0,
+    "commonvoice_hours":267.0,
     "commonvoice_locale":"lv",
     "in_benchmark":true
   },
@@ -3547,7 +3547,7 @@
     "family":"Abkhaz-Adyge",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":268.0,
+    "commonvoice_hours":271.0,
     "commonvoice_locale":"kbd",
     "in_benchmark":false
   },
@@ -3667,7 +3667,7 @@
     "family":"Indo-European",
     "flores_path":"ydd_Hebr",
     "fleurs_tag":null,
-    "commonvoice_hours":2.0,
+    "commonvoice_hours":2.1,
     "commonvoice_locale":"yi",
     "in_benchmark":true
   },
@@ -3763,8 +3763,8 @@
     "family":"Austroasiatic",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":0.0,
+    "commonvoice_locale":"mnw",
     "in_benchmark":false
   },
   {
@@ -4099,7 +4099,7 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":66.0,
+    "commonvoice_hours":67.0,
     "commonvoice_locale":"fy-NL",
     "in_benchmark":false
   },
@@ -4339,7 +4339,7 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":32.0,
+    "commonvoice_hours":33.0,
     "commonvoice_locale":"br",
     "in_benchmark":false
   },
@@ -4579,7 +4579,7 @@
     "family":"Afro-Asiatic",
     "flores_path":"mlt_Latn",
     "fleurs_tag":"mt_mt",
-    "commonvoice_hours":8.7,
+    "commonvoice_hours":8.8,
     "commonvoice_locale":"mt",
     "in_benchmark":true
   },
@@ -4639,7 +4639,7 @@
     "family":"Abkhaz-Adyge",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":71.0,
+    "commonvoice_hours":76.0,
     "commonvoice_locale":"ady",
     "in_benchmark":false
   },
@@ -4999,7 +4999,7 @@
     "family":"Nakh-Daghestanian",
     "flores_path":"dar_Cyrl",
     "fleurs_tag":null,
-    "commonvoice_hours":15.0,
+    "commonvoice_hours":16.0,
     "commonvoice_locale":"dar",
     "in_benchmark":true
   },
@@ -5779,8 +5779,8 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":0.0,
+    "commonvoice_locale":"swb",
     "in_benchmark":false
   },
   {
@@ -6211,7 +6211,7 @@
     "family":"Abkhaz-Adyge",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":216.0,
+    "commonvoice_hours":217.0,
     "commonvoice_locale":"ab",
     "in_benchmark":false
   },
@@ -6871,7 +6871,7 @@
     "family":"Kartvelian",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":26.0,
+    "commonvoice_hours":40.0,
     "commonvoice_locale":"lzz",
     "in_benchmark":false
   },
@@ -7039,7 +7039,7 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":3.8,
+    "commonvoice_hours":3.9,
     "commonvoice_locale":"hsb",
     "in_benchmark":false
   },
@@ -7867,7 +7867,7 @@
     "family":"Artificial Language",
     "flores_path":"epo_Latn",
     "fleurs_tag":null,
-    "commonvoice_hours":1440.0,
+    "commonvoice_hours":1441.0,
     "commonvoice_locale":"eo",
     "in_benchmark":true
   },

results/model_health.json

@@ -0,0 +1,520 @@
+[
+  {
+    "model":"z-ai\/glm-4.6",
+    "total":15030,
+    "failed":10109,
+    "score_nonfailed":0.8472325073,
+    "failed_pct":67.2588157019
+  },
+  {
+    "model":"minimax\/minimax-m2.5",
+    "total":15030,
+    "failed":7130,
+    "score_nonfailed":0.716766251,
+    "failed_pct":47.4384564205
+  },
+  {
+    "model":"qwen\/qwen3-235b-a22b",
+    "total":15030,
+    "failed":6075,
+    "score_nonfailed":0.5493210221,
+    "failed_pct":40.4191616766
+  },
+  {
+    "model":"google\/translate-v2",
+    "total":7760,
+    "failed":1400,
+    "score_nonfailed":0.4370955661,
+    "failed_pct":18.0412371134
+  },
+  {
+    "model":"qwen\/qwen3-30b-a3b",
+    "total":15030,
+    "failed":2186,
+    "score_nonfailed":0.4860680279,
+    "failed_pct":14.5442448436
+  },
+  {
+    "model":"anthropic\/claude-sonnet-4.6",
+    "total":15030,
+    "failed":1193,
+    "score_nonfailed":0.6553656376,
+    "failed_pct":7.9374584165
+  },
+  {
+    "model":"x-ai\/grok-4.1-fast",
+    "total":15030,
+    "failed":1192,
+    "score_nonfailed":0.5416815078,
+    "failed_pct":7.9308050566
+  },
+  {
+    "model":"openai\/gpt-5",
+    "total":15030,
+    "failed":1175,
+    "score_nonfailed":0.6563698813,
+    "failed_pct":7.8176979375
+  },
+  {
+    "model":"allenai\/olmo-3.1-32b-instruct",
+    "total":15030,
+    "failed":947,
+    "score_nonfailed":0.3859660101,
+    "failed_pct":6.3007318696
+  },
+  {
+    "model":"deepseek\/deepseek-v3.2-exp",
+    "total":15030,
+    "failed":920,
+    "score_nonfailed":0.5855771228,
+    "failed_pct":6.121091151
+  },
+  {
+    "model":"anthropic\/claude-opus-4.6",
+    "total":15030,
+    "failed":536,
+    "score_nonfailed":0.6793265387,
+    "failed_pct":3.5662009315
+  },
+  {
+    "model":"google\/gemini-3.1-pro-preview",
+    "total":15030,
+    "failed":534,
+    "score_nonfailed":0.6915737079,
+    "failed_pct":3.5528942116
+  },
+  {
+    "model":"openai\/gpt-5-nano",
+    "total":15030,
+    "failed":453,
+    "score_nonfailed":0.5038289898,
+    "failed_pct":3.0139720559
+  },
+  {
+    "model":"openai\/gpt-5.4",
+    "total":15030,
+    "failed":247,
+    "score_nonfailed":0.6320077882,
+    "failed_pct":1.6433799069
+  },
+  {
+    "model":"openai\/gpt-5.2",
+    "total":15030,
+    "failed":96,
+    "score_nonfailed":0.6242762659,
+    "failed_pct":0.6387225549
+  },
+  {
+    "model":"openai\/gpt-5-mini",
+    "total":15030,
+    "failed":81,
+    "score_nonfailed":0.5571339927,
+    "failed_pct":0.5389221557
+  },
+  {
+    "model":"openai\/gpt-5.1",
+    "total":15030,
+    "failed":35,
+    "score_nonfailed":0.6182095927,
+    "failed_pct":0.2328675981
+  },
+  {
+    "model":"anthropic\/claude-opus-4.5",
+    "total":15030,
+    "failed":27,
+    "score_nonfailed":0.6613529281,
+    "failed_pct":0.1796407186
+  },
+  {
+    "model":"anthropic\/claude-sonnet-4.5",
+    "total":15030,
+    "failed":19,
+    "score_nonfailed":0.644467649,
+    "failed_pct":0.126413839
+  },
+  {
+    "model":"amazon\/nova-pro-v1",
+    "total":15030,
+    "failed":8,
+    "score_nonfailed":0.5459768526,
+    "failed_pct":0.0532268796
+  },
+  {
+    "model":"openai\/gpt-oss-120b",
+    "total":15030,
+    "failed":8,
+    "score_nonfailed":0.5131642507,
+    "failed_pct":0.0532268796
+  },
+  {
+    "model":"mistralai\/mistral-nemo",
+    "total":15030,
+    "failed":1,
+    "score_nonfailed":0.3401549906,
+    "failed_pct":0.0066533599
+  },
+  {
+    "model":"qwen\/qwen3-vl-30b-a3b-thinking",
+    "total":17,
+    "failed":0,
+    "score_nonfailed":0.5794774559,
+    "failed_pct":0.0
+  },
+  {
+    "model":"anthropic\/claude-3-7-sonnet-20250219",
+    "total":2818,
+    "failed":0,
+    "score_nonfailed":0.6793719385,
+    "failed_pct":0.0
+  },
+  {
+    "model":"openai\/gpt-3.5-turbo-0613",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.5504296912,
+    "failed_pct":0.0
+  },
+  {
+    "model":"z-ai\/glm-4.5-air",
+    "total":140,
+    "failed":0,
+    "score_nonfailed":0.582694482,
+    "failed_pct":0.0
+  },
+  {
+    "model":"openai\/gpt-4.1",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.5585604761,
+    "failed_pct":0.0
+  },
+  {
+    "model":"openai\/gpt-4.1-mini",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.6424898144,
+    "failed_pct":0.0
+  },
+  {
+    "model":"openai\/gpt-4.1-nano",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.6064425983,
+    "failed_pct":0.0
+  },
+  {
+    "model":"openai\/gpt-4o",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.5881047852,
+    "failed_pct":0.0
+  },
+  {
+    "model":"openai\/gpt-4o-2024-11-20",
+    "total":17,
+    "failed":0,
+    "score_nonfailed":0.742557214,
+    "failed_pct":0.0
+  },
+  {
+    "model":"openai\/gpt-4o-mini",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.6541716124,
+    "failed_pct":0.0
+  },
+  {
+    "model":"openai\/gpt-5-codex",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.7040281767,
+    "failed_pct":0.0
+  },
+  {
+    "model":"openai\/gpt-5-chat-2025-08-07",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.639942047,
+    "failed_pct":0.0
+  },
+  {
+    "model":"qwen\/qwen3-8b-04-28",
+    "total":113,
+    "failed":0,
+    "score_nonfailed":0.6392222245,
+    "failed_pct":0.0
+  },
+  {
+    "model":"amazon\/nova-micro-v1",
+    "total":17,
+    "failed":0,
+    "score_nonfailed":0.6700323243,
+    "failed_pct":0.0
+  },
+  {
+    "model":"x-ai\/grok-4-fast",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.5469999688,
+    "failed_pct":0.0
+  },
+  {
+    "model":"x-ai\/grok-4",
+    "total":3475,
+    "failed":0,
+    "score_nonfailed":0.6712118876,
+    "failed_pct":0.0
+  },
+  {
+    "model":"nousresearch\/hermes-3-llama-3.1-405b",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.6629729711,
+    "failed_pct":0.0
+  },
+  {
+    "model":"x-ai\/grok-3",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.6330974229,
+    "failed_pct":0.0
+  },
+  {
+    "model":"openai\/o4-mini-2025-04-16",
+    "total":167,
+    "failed":0,
+    "score_nonfailed":0.646647725,
+    "failed_pct":0.0
+  },
+  {
+    "model":"perplexity\/sonar-reasoning-pro",
+    "total":17,
+    "failed":0,
+    "score_nonfailed":0.7141204358,
+    "failed_pct":0.0
+  },
+  {
+    "model":"amazon\/nova-premier-v1",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.5273180978,
+    "failed_pct":0.0
+  },
+  {
+    "model":"openai\/gpt-3.5-turbo",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.3505272925,
+    "failed_pct":0.0
+  },
+  {
+    "model":"mistralai\/mixtral-8x7b-instruct",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.4240249599,
+    "failed_pct":0.0
+  },
+  {
+    "model":"neversleep\/noromaid-20b",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.2124753832,
+    "failed_pct":0.0
+  },
+  {
+    "model":"google\/gemma-3-27b-it",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.5467921901,
+    "failed_pct":0.0
+  },
+  {
+    "model":"anthropic\/claude-sonnet-4",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.6243490094,
+    "failed_pct":0.0
+  },
+  {
+    "model":"anthropic\/claude-haiku-4.5",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.5391267264,
+    "failed_pct":0.0
+  },
+  {
+    "model":"baidu\/ernie-4.5-300b-a47b",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.5751447662,
+    "failed_pct":0.0
+  },
+  {
+    "model":"cohere\/command-a",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.5053932299,
+    "failed_pct":0.0
+  },
+  {
+    "model":"cohere\/command-r-08-2024",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.5222153257,
+    "failed_pct":0.0
+  },
+  {
+    "model":"cohere\/command-r-plus-08-2024",
+    "total":6369,
+    "failed":0,
+    "score_nonfailed":0.3479812964,
+    "failed_pct":0.0
+  },
+  {
+    "model":"google\/gemini-2.0-flash-lite-001",
+    "total":17,
+    "failed":0,
+    "score_nonfailed":0.7041712433,
+    "failed_pct":0.0
+  },
+  {
+    "model":"google\/gemini-2.5-flash",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.4237452366,
+    "failed_pct":0.0
+  },
+  {
+    "model":"google\/gemini-2.5-flash-image",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.713401524,
+    "failed_pct":0.0
+  },
+  {
+    "model":"google\/gemini-2.5-flash-lite",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.5594649326,
+    "failed_pct":0.0
+  },
+  {
+    "model":"google\/gemini-2.5-pro",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.6415432467,
+    "failed_pct":0.0
+  },
+  {
+    "model":"google\/gemini-3-pro-preview",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.6580387071,
+    "failed_pct":0.0
+  },
+  {
+    "model":"anthropic\/claude-3.7-sonnet",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.632849603,
+    "failed_pct":0.0
+  },
+  {
+    "model":"moonshotai\/kimi-k2",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.55318705,
+    "failed_pct":0.0
+  },
+  {
+    "model":"meta-llama\/llama-3-70b-instruct",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.468044074,
+    "failed_pct":0.0
+  },
+  {
+    "model":"meta-llama\/llama-3.1-70b-instruct",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.4860431126,
+    "failed_pct":0.0
+  },
+  {
+    "model":"meta-llama\/llama-3.3-70b-instruct",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.4846053463,
+    "failed_pct":0.0
+  },
+  {
+    "model":"meta-llama\/llama-4-maverick",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.5585725974,
+    "failed_pct":0.0
+  },
+  {
+    "model":"microsoft\/phi-3-mini-128k-instruct",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.3182626307,
+    "failed_pct":0.0
+  },
+  {
+    "model":"microsoft\/phi-4",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.3454741484,
+    "failed_pct":0.0
+  },
+  {
+    "model":"microsoft\/phi-4-multimodal-instruct",
+    "total":17,
+    "failed":0,
+    "score_nonfailed":0.7358366016,
+    "failed_pct":0.0
+  },
+  {
+    "model":"anthropic\/claude-3.5-sonnet",
+    "total":14288,
+    "failed":0,
+    "score_nonfailed":0.6279979071,
+    "failed_pct":0.0
+  },
+  {
+    "model":"mistralai\/magistral-small-2506",
+    "total":120,
+    "failed":0,
+    "score_nonfailed":0.3314969313,
+    "failed_pct":0.0
+  },
+  {
+    "model":"alpindale\/goliath-120b",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.3700368906,
+    "failed_pct":0.0
+  },
+  {
+    "model":"mistralai\/mistral-saba",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.4565603305,
+    "failed_pct":0.0
+  },
+  {
+    "model":"mistralai\/mistral-small-3.1-24b-instruct",
+    "total":17,
+    "failed":0,
+    "score_nonfailed":0.70419565,
+    "failed_pct":0.0
+  },
+  {
+    "model":"mistralai\/mistral-medium-3.1",
+    "total":15030,
+    "failed":0,
+    "score_nonfailed":0.5117549741,
+    "failed_pct":0.0
+  }
+]

results/models.json

@@ -1,15 +1,15 @@
 [
   {
-    "id":"allenai\/olmo-3.1-32b-instruct",
-    "name":"Olmo 3.1 32B Instruct",
-    "provider_name":"AllenAI",
-    "cost":0.6,
+    "id":"ai21\/jamba-large-1.7",
+    "name":"Jamba Large 1.7",
+    "provider_name":"AI21",
+    "cost":8.0,
     "train_on_prompts":false,
-    "hf_id":"allenai\/Olmo-3.1-32B-Instruct",
+    "hf_id":"ai21labs\/AI21-Jamba-Large-1.7",
     "size":null,
     "type":"open-source",
-    "license":"Apache 2.0",
-    "creation_date":1765324800000,
+    "license":"Other",
+    "creation_date":1751414400000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -21,16 +21,16 @@
     ]
   },
   {
-    "id":"amazon\/nova-premier-v1",
-    "name":"Nova Premier 1.0",
+    "id":"amazon\/nova-2-lite-v1",
+    "name":"Nova 2 Lite",
     "provider_name":"Amazon",
-    "cost":12.5,
+    "cost":2.5,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1761868800000,
+    "creation_date":1764633600000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -42,16 +42,16 @@
     ]
   },
   {
-    "id":"amazon\/nova-pro-v1",
-    "name":"Nova Pro 1.0",
+    "id":"amazon\/nova-premier-v1",
+    "name":"Nova Premier 1.0",
     "provider_name":"Amazon",
-    "cost":3.2,
+    "cost":12.5,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1733356800000,
+    "creation_date":1761868800000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -63,16 +63,16 @@
     ]
   },
   {
-    "id":"anthropic\/claude-3.7-sonnet",
-    "name":"Claude 3.7 Sonnet",
-    "provider_name":"Anthropic",
-    "cost":15.0,
+    "id":"amazon\/nova-pro-v1",
+    "name":"Nova Pro 1.0",
+    "provider_name":"Amazon",
+    "cost":3.2,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1740355200000,
+    "creation_date":1733356800000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -146,6 +146,27 @@
       "mgsm"
     ]
   },
+  {
+    "id":"anthropic\/claude-opus-4.7",
+    "name":"Claude Opus 4.7",
+    "provider_name":"Anthropic",
+    "cost":25.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1776297600000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
   {
     "id":"anthropic\/claude-sonnet-4",
     "name":"Claude Sonnet 4",
@@ -210,16 +231,16 @@
     ]
   },
   {
-    "id":"baidu\/ernie-4.5-300b-a47b",
-    "name":"ERNIE 4.5 300B A47B ",
-    "provider_name":"Baidu",
-    "cost":1.1,
+    "id":"arcee-ai\/trinity-mini-20251201",
+    "name":"Trinity Mini",
+    "provider_name":"Arcee AI",
+    "cost":0.15,
     "train_on_prompts":false,
-    "hf_id":"baidu\/ERNIE-4.5-300B-A47B-PT",
+    "hf_id":"arcee-ai\/Trinity-Mini",
     "size":null,
     "type":"open-source",
     "license":"Apache 2.0",
-    "creation_date":1751068800000,
+    "creation_date":1764547200000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -231,16 +252,16 @@
     ]
   },
   {
-    "id":"cohere\/command-a",
-    "name":"Command A",
-    "provider_name":"Cohere",
-    "cost":10.0,
+    "id":"baidu\/ernie-4.5-21b-a3b",
+    "name":"ERNIE 4.5 21B A3B",
+    "provider_name":"Baidu",
+    "cost":0.28,
     "train_on_prompts":false,
-    "hf_id":"CohereLabs\/c4ai-command-a-03-2025",
+    "hf_id":"baidu\/ERNIE-4.5-21B-A3B-PT",
     "size":null,
     "type":"open-source",
-    "license":"Cc By Nc 4.0",
-    "creation_date":1741651200000,
+    "license":"Apache 2.0",
+    "creation_date":1751068800000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -252,16 +273,16 @@
     ]
   },
   {
-    "id":"deepseek\/deepseek-v3.2-exp",
-    "name":"DeepSeek V3.2 Exp",
-    "provider_name":"DeepSeek",
-    "cost":0.41,
+    "id":"baidu\/ernie-4.5-300b-a47b",
+    "name":"ERNIE 4.5 300B A47B ",
+    "provider_name":"Baidu",
+    "cost":1.1,
     "train_on_prompts":false,
-    "hf_id":"deepseek-ai\/DeepSeek-V3.2-Exp",
+    "hf_id":"baidu\/ERNIE-4.5-300B-A47B-PT",
     "size":null,
     "type":"open-source",
-    "license":"Mit",
-    "creation_date":1759104000000,
+    "license":"Apache 2.0",
+    "creation_date":1751068800000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -273,16 +294,16 @@
     ]
   },
   {
-    "id":"google\/gemini-2.5-flash",
-    "name":"Gemini 2.5 Flash",
-    "provider_name":"Google",
-    "cost":2.5,
+    "id":"bytedance-seed\/seed-1.6-20250625",
+    "name":"Seed 1.6",
+    "provider_name":"ByteDance Seed",
+    "cost":2.0,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1750118400000,
+    "creation_date":1766448000000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -294,16 +315,16 @@
     ]
   },
   {
-    "id":"google\/gemini-2.5-flash-lite",
-    "name":"Gemini 2.5 Flash Lite",
-    "provider_name":"Google",
-    "cost":0.4,
+    "id":"bytedance-seed\/seed-1.6-flash-20250625",
+    "name":"Seed 1.6 Flash",
+    "provider_name":"ByteDance Seed",
+    "cost":0.3,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1753142400000,
+    "creation_date":1766448000000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -315,16 +336,16 @@
     ]
   },
   {
-    "id":"google\/gemini-2.5-pro",
-    "name":"Gemini 2.5 Pro",
-    "provider_name":"Google",
-    "cost":10.0,
+    "id":"bytedance-seed\/seed-2.0-lite-20260309",
+    "name":"Seed-2.0-Lite",
+    "provider_name":"ByteDance Seed",
+    "cost":2.0,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1750118400000,
+    "creation_date":1773100800000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -336,16 +357,16 @@
     ]
   },
   {
-    "id":"google\/gemini-3-pro-preview",
-    "name":"Gemini 3 Pro Preview",
-    "provider_name":"Google",
-    "cost":12.0,
+    "id":"bytedance-seed\/seed-2.0-mini-20260224",
+    "name":"Seed-2.0-Mini",
+    "provider_name":"ByteDance Seed",
+    "cost":0.4,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1763424000000,
+    "creation_date":1772064000000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -357,16 +378,37 @@
     ]
   },
   {
-    "id":"google\/gemini-3.1-pro-preview",
-    "name":"Gemini 3.1 Pro Preview",
-    "provider_name":"Google",
-    "cost":12.0,
+    "id":"cohere\/command-a",
+    "name":"Command A",
+    "provider_name":"Cohere",
+    "cost":10.0,
+    "train_on_prompts":false,
+    "hf_id":"CohereLabs\/c4ai-command-a-03-2025",
+    "size":111057580032.0,
+    "type":"open-source",
+    "license":"Cc By Nc 4.0",
+    "creation_date":1741651200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"deepcogito\/cogito-v2.1-671b-20251118",
+    "name":"Cogito v2.1 671B",
+    "provider_name":"Deep Cogito",
+    "cost":1.25,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1771459200000,
+    "creation_date":1762992000000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -378,16 +420,16 @@
     ]
   },
   {
-    "id":"meta-llama\/llama-3-70b-instruct",
-    "name":"Llama 3 70B Instruct",
-    "provider_name":"Meta",
-    "cost":0.74,
+    "id":"deepseek\/deepseek-chat-v3.1",
+    "name":"DeepSeek V3.1",
+    "provider_name":"DeepSeek",
+    "cost":0.79,
     "train_on_prompts":false,
-    "hf_id":"meta-llama\/Meta-Llama-3-70B-Instruct",
-    "size":70553706496.0,
+    "hf_id":"deepseek-ai\/DeepSeek-V3.1",
+    "size":684531386000.0,
     "type":"open-source",
-    "license":"Llama3",
-    "creation_date":1713312000000,
+    "license":"Mit",
+    "creation_date":1755734400000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -399,16 +441,16 @@
     ]
   },
   {
-    "id":"meta-llama\/llama-3.1-70b-instruct",
-    "name":"Llama 3.1 70B Instruct",
-    "provider_name":"Meta",
-    "cost":0.4,
+    "id":"deepseek\/deepseek-r1-0528",
+    "name":"R1 0528",
+    "provider_name":"DeepSeek",
+    "cost":2.15,
     "train_on_prompts":false,
-    "hf_id":"meta-llama\/Llama-3.1-70B-Instruct",
-    "size":70553706496.0,
+    "hf_id":"deepseek-ai\/DeepSeek-R1-0528",
+    "size":684531386000.0,
     "type":"open-source",
-    "license":"Llama3.1",
-    "creation_date":1721088000000,
+    "license":"Mit",
+    "creation_date":1748390400000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -420,16 +462,16 @@
     ]
   },
   {
-    "id":"meta-llama\/llama-3.3-70b-instruct",
-    "name":"Llama 3.3 70B Instruct",
-    "provider_name":"Meta",
-    "cost":0.32,
+    "id":"deepseek\/deepseek-v3.1-terminus",
+    "name":"DeepSeek V3.1 Terminus",
+    "provider_name":"DeepSeek",
+    "cost":0.95,
     "train_on_prompts":false,
-    "hf_id":"meta-llama\/Llama-3.3-70B-Instruct",
-    "size":null,
+    "hf_id":"deepseek-ai\/DeepSeek-V3.1-Terminus",
+    "size":684531386000.0,
     "type":"open-source",
-    "license":"Llama3.3",
-    "creation_date":1732579200000,
+    "license":"Mit",
+    "creation_date":1758499200000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -441,16 +483,16 @@
     ]
   },
   {
-    "id":"meta-llama\/llama-4-maverick",
-    "name":"Llama 4 Maverick",
-    "provider_name":"Meta",
-    "cost":0.6,
+    "id":"deepseek\/deepseek-v3.2-exp",
+    "name":"DeepSeek V3.2 Exp",
+    "provider_name":"DeepSeek",
+    "cost":0.41,
     "train_on_prompts":false,
-    "hf_id":"meta-llama\/Llama-4-Maverick-17B-128E-Instruct",
-    "size":401583781376.0,
+    "hf_id":"deepseek-ai\/DeepSeek-V3.2-Exp",
+    "size":685396921376.0,
     "type":"open-source",
-    "license":"Other",
-    "creation_date":1743465600000,
+    "license":"Mit",
+    "creation_date":1759104000000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -462,16 +504,16 @@
     ]
   },
   {
-    "id":"microsoft\/phi-4",
-    "name":"Phi 4",
-    "provider_name":"Microsoft",
-    "cost":0.14,
+    "id":"deepseek\/deepseek-v3.2-speciale-20251201",
+    "name":"DeepSeek V3.2 Speciale",
+    "provider_name":"DeepSeek",
+    "cost":0.43,
     "train_on_prompts":false,
-    "hf_id":"microsoft\/phi-4",
+    "hf_id":"deepseek-ai\/DeepSeek-V3.2-Speciale",
     "size":null,
     "type":"open-source",
     "license":"Mit",
-    "creation_date":1733875200000,
+    "creation_date":1764288000000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -483,16 +525,16 @@
     ]
   },
   {
-    "id":"minimax\/minimax-m2.5",
-    "name":"MiniMax M2.5",
-    "provider_name":"MiniMax",
-    "cost":1.2,
+    "id":"deepseek\/deepseek-v4-flash-20260423",
+    "name":"DeepSeek V4 Flash",
+    "provider_name":"DeepSeek",
+    "cost":0.22,
     "train_on_prompts":false,
-    "hf_id":"MiniMaxAI\/MiniMax-M2.5",
-    "size":228703644928.0,
+    "hf_id":"deepseek-ai\/DeepSeek-V4-Flash",
+    "size":158069433298.0,
     "type":"open-source",
-    "license":"Other",
-    "creation_date":1770854400000,
+    "license":"Mit",
+    "creation_date":1776816000000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -504,16 +546,16 @@
     ]
   },
   {
-    "id":"mistralai\/mistral-medium-3.1",
-    "name":"Mistral Medium 3.1",
-    "provider_name":"Mistral",
-    "cost":2.0,
+    "id":"google\/gemini-2.5-flash",
+    "name":"Gemini 2.5 Flash",
+    "provider_name":"Google",
+    "cost":2.5,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1755043200000,
+    "creation_date":1750118400000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -525,16 +567,16 @@
     ]
   },
   {
-    "id":"mistralai\/mistral-nemo",
-    "name":"Mistral Nemo",
-    "provider_name":"Mistral",
-    "cost":0.04,
+    "id":"google\/gemini-2.5-flash-lite",
+    "name":"Gemini 2.5 Flash Lite",
+    "provider_name":"Google",
+    "cost":0.4,
     "train_on_prompts":false,
-    "hf_id":"mistralai\/Mistral-Nemo-Instruct-2407",
+    "hf_id":null,
     "size":null,
-    "type":"open-source",
-    "license":"Apache 2.0",
-    "creation_date":1721174400000,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1753142400000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -546,16 +588,16 @@
     ]
   },
   {
-    "id":"mistralai\/mistral-saba",
-    "name":"Saba",
-    "provider_name":"Mistral",
-    "cost":0.6,
+    "id":"google\/gemini-2.5-pro",
+    "name":"Gemini 2.5 Pro",
+    "provider_name":"Google",
+    "cost":10.0,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1739750400000,
+    "creation_date":1750118400000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -567,16 +609,16 @@
     ]
   },
   {
-    "id":"moonshotai\/kimi-k2",
-    "name":"Kimi K2 0711",
-    "provider_name":"MoonshotAI",
-    "cost":2.2,
+    "id":"google\/gemini-3.1-flash-lite",
+    "name":"Gemini 3.1 Flash Lite",
+    "provider_name":"Google",
+    "cost":1.5,
     "train_on_prompts":false,
-    "hf_id":"moonshotai\/Kimi-K2-Instruct",
-    "size":1026470731056.0,
-    "type":"open-source",
-    "license":"Other",
-    "creation_date":1752192000000,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1778112000000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -588,16 +630,16 @@
     ]
   },
   {
-    "id":"openai\/gpt-3.5-turbo",
-    "name":"GPT-3.5 Turbo",
-    "provider_name":"OpenAI",
-    "cost":1.5,
+    "id":"google\/gemini-3.1-pro-preview",
+    "name":"Gemini 3.1 Pro Preview",
+    "provider_name":"Google",
+    "cost":12.0,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1685232000000,
+    "creation_date":1771459200000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -609,16 +651,16 @@
     ]
   },
   {
-    "id":"openai\/gpt-4.1",
-    "name":"GPT-4.1",
-    "provider_name":"OpenAI",
-    "cost":8.0,
+    "id":"google\/gemma-3-27b-it",
+    "name":"Gemma 3 27B",
+    "provider_name":"Google",
+    "cost":0.16,
     "train_on_prompts":false,
-    "hf_id":null,
-    "size":null,
-    "type":"closed-source",
-    "license":null,
-    "creation_date":1744588800000,
+    "hf_id":"google\/gemma-3-27b-it",
+    "size":27432406640.0,
+    "type":"open-source",
+    "license":"Gemma",
+    "creation_date":1740787200000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -630,16 +672,16 @@
     ]
   },
   {
-    "id":"openai\/gpt-4o",
-    "name":"GPT-4o",
-    "provider_name":"OpenAI",
-    "cost":10.0,
+    "id":"google\/gemma-3n-e4b-it",
+    "name":"Gemma 3n 4B",
+    "provider_name":"Google",
+    "cost":0.12,
     "train_on_prompts":false,
-    "hf_id":null,
+    "hf_id":"google\/gemma-3n-E4B-it",
     "size":null,
-    "type":"closed-source",
-    "license":null,
-    "creation_date":1715558400000,
+    "type":"open-source",
+    "license":"Gemma",
+    "creation_date":1748908800000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -651,7 +693,658 @@
     ]
   },
   {
-    "id":"openai\/gpt-5",
+    "id":"google\/gemma-4-26b-a4b-it-20260403",
+    "name":"Gemma 4 26B A4B ",
+    "provider_name":"Google",
+    "cost":0.33,
+    "train_on_prompts":false,
+    "hf_id":"google\/gemma-4-26B-A4B-it",
+    "size":26544131376.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1773187200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"google\/gemma-4-31b-it-20260402",
+    "name":"Gemma 4 31B",
+    "provider_name":"Google",
+    "cost":0.37,
+    "train_on_prompts":false,
+    "hf_id":"google\/gemma-4-31B-it",
+    "size":32682372656.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1773187200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"ibm-granite\/granite-4.0-h-micro",
+    "name":"Granite 4.0 Micro",
+    "provider_name":"IBM",
+    "cost":0.11,
+    "train_on_prompts":false,
+    "hf_id":"ibm-granite\/granite-4.0-h-micro",
+    "size":3191396096.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1757980800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"ibm-granite\/granite-4.1-8b-20260429",
+    "name":"Granite 4.1 8B",
+    "provider_name":"IBM",
+    "cost":0.1,
+    "train_on_prompts":false,
+    "hf_id":"ibm-granite\/granite-4.1-8b",
+    "size":8791592960.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1775433600000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"liquid\/lfm-2-24b-a2b-20260224",
+    "name":"LFM2-24B-A2B",
+    "provider_name":"LiquidAI",
+    "cost":0.12,
+    "train_on_prompts":false,
+    "hf_id":"LiquidAI\/LFM2-24B-A2B",
+    "size":23843661440.0,
+    "type":"open-source",
+    "license":"Other",
+    "creation_date":1771891200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"meta-llama\/llama-3-70b-instruct",
+    "name":"Llama 3 70B Instruct",
+    "provider_name":"Meta",
+    "cost":0.74,
+    "train_on_prompts":false,
+    "hf_id":"meta-llama\/Meta-Llama-3-70B-Instruct",
+    "size":70553706496.0,
+    "type":"open-source",
+    "license":"Llama3",
+    "creation_date":1713312000000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"meta-llama\/llama-3.1-70b-instruct",
+    "name":"Llama 3.1 70B Instruct",
+    "provider_name":"Meta",
+    "cost":0.4,
+    "train_on_prompts":false,
+    "hf_id":"meta-llama\/Llama-3.1-70B-Instruct",
+    "size":70553706496.0,
+    "type":"open-source",
+    "license":"Llama3.1",
+    "creation_date":1721088000000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"meta-llama\/llama-3.3-70b-instruct",
+    "name":"Llama 3.3 70B Instruct",
+    "provider_name":"Meta",
+    "cost":0.32,
+    "train_on_prompts":false,
+    "hf_id":"meta-llama\/Llama-3.3-70B-Instruct",
+    "size":70553706496.0,
+    "type":"open-source",
+    "license":"Llama3.3",
+    "creation_date":1732579200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"meta-llama\/llama-4-maverick",
+    "name":"Llama 4 Maverick",
+    "provider_name":"Meta",
+    "cost":0.6,
+    "train_on_prompts":false,
+    "hf_id":"meta-llama\/Llama-4-Maverick-17B-128E-Instruct",
+    "size":401583781376.0,
+    "type":"open-source",
+    "license":"Other",
+    "creation_date":1743465600000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"microsoft\/phi-4",
+    "name":"Phi 4",
+    "provider_name":"Microsoft",
+    "cost":0.14,
+    "train_on_prompts":false,
+    "hf_id":"microsoft\/phi-4",
+    "size":null,
+    "type":"open-source",
+    "license":"Mit",
+    "creation_date":1733875200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"microsoft\/phi-4-mini-instruct",
+    "name":"Phi 4 Mini Instruct",
+    "provider_name":"Microsoft",
+    "cost":0.35,
+    "train_on_prompts":false,
+    "hf_id":"microsoft\/Phi-4-mini-instruct",
+    "size":null,
+    "type":"open-source",
+    "license":"Mit",
+    "creation_date":1739923200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"mistralai\/ministral-14b-2512",
+    "name":"Ministral 3 14B 2512",
+    "provider_name":"Mistral",
+    "cost":0.2,
+    "train_on_prompts":false,
+    "hf_id":"mistralai\/Ministral-3-14B-Instruct-2512",
+    "size":null,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1761868800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"mistralai\/ministral-3b-2512",
+    "name":"Ministral 3 3B 2512",
+    "provider_name":"Mistral",
+    "cost":0.1,
+    "train_on_prompts":false,
+    "hf_id":"mistralai\/Ministral-3-3B-Instruct-2512",
+    "size":null,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1761868800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"mistralai\/ministral-8b-2512",
+    "name":"Ministral 3 8B 2512",
+    "provider_name":"Mistral",
+    "cost":0.15,
+    "train_on_prompts":false,
+    "hf_id":"mistralai\/Ministral-3-8B-Instruct-2512",
+    "size":8918026716.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1761868800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"mistralai\/mistral-large-2512",
+    "name":"Mistral Large 3 2512",
+    "provider_name":"Mistral",
+    "cost":1.5,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1764547200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"mistralai\/mistral-medium-3.1",
+    "name":"Mistral Medium 3.1",
+    "provider_name":"Mistral",
+    "cost":2.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1755043200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"mistralai\/mistral-nemo",
+    "name":"Mistral Nemo",
+    "provider_name":"Mistral",
+    "cost":0.03,
+    "train_on_prompts":false,
+    "hf_id":"mistralai\/Mistral-Nemo-Instruct-2407",
+    "size":null,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1721174400000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"mistralai\/mistral-saba",
+    "name":"Saba",
+    "provider_name":"Mistral",
+    "cost":0.6,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1739750400000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"mistralai\/mistral-small-2603",
+    "name":"Mistral Small 4",
+    "provider_name":"Mistral",
+    "cost":0.6,
+    "train_on_prompts":false,
+    "hf_id":"mistralai\/Mistral-Small-4-119B-2603",
+    "size":119401317952.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1769126400000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"mistralai\/mistral-small-3.2-24b-instruct",
+    "name":"Mistral Small 3.2 24B",
+    "provider_name":"Mistral",
+    "cost":0.2,
+    "train_on_prompts":false,
+    "hf_id":"mistralai\/Mistral-Small-3.2-24B-Instruct-2506",
+    "size":null,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1750291200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"moonshotai\/kimi-k2",
+    "name":"Kimi K2 0711",
+    "provider_name":"MoonshotAI",
+    "cost":2.3,
+    "train_on_prompts":false,
+    "hf_id":"moonshotai\/Kimi-K2-Instruct",
+    "size":1026470731056.0,
+    "type":"open-source",
+    "license":"Other",
+    "creation_date":1752192000000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"moonshotai\/kimi-k2.6",
+    "name":"Kimi K2.6",
+    "provider_name":"MoonshotAI",
+    "cost":3.49,
+    "train_on_prompts":false,
+    "hf_id":"moonshotai\/Kimi-K2.6",
+    "size":1058589420528.0,
+    "type":"open-source",
+    "license":"Other",
+    "creation_date":1776124800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"nousresearch\/hermes-4-405b",
+    "name":"Hermes 4 405B",
+    "provider_name":"Nous",
+    "cost":3.0,
+    "train_on_prompts":false,
+    "hf_id":"NousResearch\/Hermes-4-405B",
+    "size":405853388800.0,
+    "type":"open-source",
+    "license":"Llama3",
+    "creation_date":1754438400000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"nousresearch\/hermes-4-70b",
+    "name":"Hermes 4 70B",
+    "provider_name":"Nous",
+    "cost":0.4,
+    "train_on_prompts":false,
+    "hf_id":"NousResearch\/Hermes-4-70B",
+    "size":null,
+    "type":"open-source",
+    "license":"Llama3",
+    "creation_date":1755475200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"nvidia\/llama-3.3-nemotron-super-49b-v1.5",
+    "name":"Llama 3.3 Nemotron Super 49B V1.5",
+    "provider_name":"NVIDIA",
+    "cost":0.4,
+    "train_on_prompts":false,
+    "hf_id":"nvidia\/Llama-3_3-Nemotron-Super-49B-v1_5",
+    "size":49867145216.0,
+    "type":"open-source",
+    "license":"Other",
+    "creation_date":1753401600000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"nvidia\/nemotron-3-nano-30b-a3b",
+    "name":"Nemotron 3 Nano 30B A3B",
+    "provider_name":"NVIDIA",
+    "cost":0.2,
+    "train_on_prompts":false,
+    "hf_id":"nvidia\/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
+    "size":31577937344.0,
+    "type":"open-source",
+    "license":"Other",
+    "creation_date":1764806400000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"nvidia\/nemotron-3-super-120b-a12b-20230311",
+    "name":"Nemotron 3 Super",
+    "provider_name":"NVIDIA",
+    "cost":0.45,
+    "train_on_prompts":false,
+    "hf_id":"nvidia\/NVIDIA-Nemotron-3-Super-120B-A12B-FP8",
+    "size":123611012096.0,
+    "type":"open-source",
+    "license":"Other",
+    "creation_date":1773100800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"nvidia\/nemotron-nano-9b-v2",
+    "name":"Nemotron Nano 9B V2",
+    "provider_name":"NVIDIA",
+    "cost":0.16,
+    "train_on_prompts":false,
+    "hf_id":"nvidia\/NVIDIA-Nemotron-Nano-9B-v2",
+    "size":8888227328.0,
+    "type":"open-source",
+    "license":"Other",
+    "creation_date":1754956800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-3.5-turbo",
+    "name":"GPT-3.5 Turbo",
+    "provider_name":"OpenAI",
+    "cost":1.5,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1685232000000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-4.1",
+    "name":"GPT-4.1",
+    "provider_name":"OpenAI",
+    "cost":8.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1744588800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-4o",
+    "name":"GPT-4o",
+    "provider_name":"OpenAI",
+    "cost":10.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1715558400000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-5",
     "name":"GPT-5",
     "provider_name":"OpenAI",
     "cost":10.0,
@@ -660,7 +1353,280 @@
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1754524800000,
+    "creation_date":1754524800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-5-mini",
+    "name":"GPT-5 Mini",
+    "provider_name":"OpenAI",
+    "cost":2.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1754524800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-5-nano",
+    "name":"GPT-5 Nano",
+    "provider_name":"OpenAI",
+    "cost":0.4,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1754524800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-5.1",
+    "name":"GPT-5.1",
+    "provider_name":"OpenAI",
+    "cost":10.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1762992000000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-5.2",
+    "name":"GPT-5.2",
+    "provider_name":"OpenAI",
+    "cost":14.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1765324800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-5.3-chat-20260303",
+    "name":"GPT-5.3 Chat",
+    "provider_name":"OpenAI",
+    "cost":14.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1772496000000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-5.4",
+    "name":"GPT-5.4",
+    "provider_name":"OpenAI",
+    "cost":15.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1772668800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-5.5",
+    "name":"GPT-5.5",
+    "provider_name":"OpenAI",
+    "cost":30.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1776988800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-oss-120b",
+    "name":"gpt-oss-120b",
+    "provider_name":"OpenAI",
+    "cost":0.18,
+    "train_on_prompts":false,
+    "hf_id":"openai\/gpt-oss-120b",
+    "size":120412337472.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1754265600000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"openai\/gpt-oss-20b",
+    "name":"gpt-oss-20b",
+    "provider_name":"OpenAI",
+    "cost":0.14,
+    "train_on_prompts":false,
+    "hf_id":"openai\/gpt-oss-20b",
+    "size":21511953984.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1754265600000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"prime-intellect\/intellect-3-20251126",
+    "name":"INTELLECT-3",
+    "provider_name":"Prime Intellect",
+    "cost":1.1,
+    "train_on_prompts":false,
+    "hf_id":"PrimeIntellect\/INTELLECT-3-FP8",
+    "size":106893249280.0,
+    "type":"open-source",
+    "license":"Mit",
+    "creation_date":1764115200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"qwen\/qwen3-30b-a3b",
+    "name":"Qwen3 30B A3B",
+    "provider_name":"Qwen",
+    "cost":0.45,
+    "train_on_prompts":false,
+    "hf_id":"Qwen\/Qwen3-30B-A3B",
+    "size":null,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1745712000000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"qwen\/qwen3-30b-a3b-instruct-2507",
+    "name":"Qwen3 30B A3B Instruct 2507",
+    "provider_name":"Qwen",
+    "cost":0.3,
+    "train_on_prompts":false,
+    "hf_id":"Qwen\/Qwen3-30B-A3B-Instruct-2507",
+    "size":null,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1753660800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"qwen\/qwen3-max",
+    "name":"Qwen3 Max",
+    "provider_name":"Qwen",
+    "cost":3.9,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1758585600000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -672,16 +1638,142 @@
     ]
   },
   {
-    "id":"openai\/gpt-5-mini",
-    "name":"GPT-5 Mini",
-    "provider_name":"OpenAI",
-    "cost":2.0,
+    "id":"qwen\/qwen3-next-80b-a3b-instruct-2509",
+    "name":"Qwen3 Next 80B A3B Instruct",
+    "provider_name":"Qwen",
+    "cost":1.1,
+    "train_on_prompts":false,
+    "hf_id":"Qwen\/Qwen3-Next-80B-A3B-Instruct",
+    "size":81324862720.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1757376000000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"qwen\/qwen3.5-122b-a10b-20260224",
+    "name":"Qwen3.5-122B-A10B",
+    "provider_name":"Qwen",
+    "cost":2.08,
+    "train_on_prompts":false,
+    "hf_id":"Qwen\/Qwen3.5-122B-A10B",
+    "size":125086497008.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1771891200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"qwen\/qwen3.5-397b-a17b-20260216",
+    "name":"Qwen3.5 397B A17B",
+    "provider_name":"Qwen",
+    "cost":2.34,
+    "train_on_prompts":false,
+    "hf_id":"Qwen\/Qwen3.5-397B-A17B",
+    "size":403397928944.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1771200000000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"qwen\/qwen3.5-9b-20260310",
+    "name":"Qwen3.5-9B",
+    "provider_name":"Qwen",
+    "cost":0.15,
+    "train_on_prompts":false,
+    "hf_id":"Qwen\/Qwen3.5-9B",
+    "size":9653104368.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1772150400000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"qwen\/qwen3.6-27b-20260422",
+    "name":"Qwen3.6 27B",
+    "provider_name":"Qwen",
+    "cost":3.2,
+    "train_on_prompts":false,
+    "hf_id":"Qwen\/Qwen3.6-27B",
+    "size":27781427952.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1776729600000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"qwen\/qwen3.6-35b-a3b-20260415",
+    "name":"Qwen3.6 35B A3B",
+    "provider_name":"Qwen",
+    "cost":1.0,
+    "train_on_prompts":false,
+    "hf_id":"Qwen\/Qwen3.6-35B-A3B",
+    "size":35951822704.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1776211200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"qwen\/qwen3.6-flash",
+    "name":"Qwen3.6 Flash",
+    "provider_name":"Qwen",
+    "cost":1.12,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1754524800000,
+    "creation_date":1777248000000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -693,16 +1785,16 @@
     ]
   },
   {
-    "id":"openai\/gpt-5-nano",
-    "name":"GPT-5 Nano",
-    "provider_name":"OpenAI",
-    "cost":0.4,
+    "id":"qwen\/qwen3.6-plus-04-02",
+    "name":"Qwen3.6 Plus",
+    "provider_name":"Qwen",
+    "cost":1.95,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1754524800000,
+    "creation_date":1775088000000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -714,16 +1806,79 @@
     ]
   },
   {
-    "id":"openai\/gpt-5.1",
-    "name":"GPT-5.1",
-    "provider_name":"OpenAI",
-    "cost":10.0,
+    "id":"rekaai\/reka-edge-2603",
+    "name":"Reka Edge",
+    "provider_name":"Reka Edge",
+    "cost":0.1,
+    "train_on_prompts":false,
+    "hf_id":"RekaAI\/reka-edge-2603",
+    "size":7128509568.0,
+    "type":"open-source",
+    "license":"Other",
+    "creation_date":1773187200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"stepfun\/step-3.5-flash",
+    "name":"Step 3.5 Flash",
+    "provider_name":"StepFun",
+    "cost":0.3,
+    "train_on_prompts":false,
+    "hf_id":"stepfun-ai\/Step-3.5-Flash",
+    "size":199384301376.0,
+    "type":"open-source",
+    "license":"Apache 2.0",
+    "creation_date":1769904000000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"tencent\/hunyuan-a13b-instruct",
+    "name":"Hunyuan A13B Instruct",
+    "provider_name":"Tencent",
+    "cost":0.57,
+    "train_on_prompts":false,
+    "hf_id":"tencent\/Hunyuan-A13B-Instruct",
+    "size":null,
+    "type":"open-source",
+    "license":"Other",
+    "creation_date":1750809600000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"upstage\/solar-pro-3",
+    "name":"Solar Pro 3",
+    "provider_name":"Upstage",
+    "cost":0.6,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1762992000000,
+    "creation_date":1769472000000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -735,16 +1890,16 @@
     ]
   },
   {
-    "id":"openai\/gpt-5.2",
-    "name":"GPT-5.2",
-    "provider_name":"OpenAI",
-    "cost":14.0,
+    "id":"writer\/palmyra-x5-20250428",
+    "name":"Palmyra X5",
+    "provider_name":"Writer",
+    "cost":6.0,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1765324800000,
+    "creation_date":1768953600000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -756,16 +1911,16 @@
     ]
   },
   {
-    "id":"openai\/gpt-5.4",
-    "name":"GPT-5.4",
-    "provider_name":"OpenAI",
-    "cost":15.0,
+    "id":"x-ai\/grok-4.20",
+    "name":"Grok 4.20",
+    "provider_name":"xAI",
+    "cost":2.5,
     "train_on_prompts":false,
     "hf_id":null,
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1772668800000,
+    "creation_date":1774915200000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -777,16 +1932,37 @@
     ]
   },
   {
-    "id":"openai\/gpt-oss-120b",
-    "name":"gpt-oss-120b",
-    "provider_name":"OpenAI",
-    "cost":0.19,
+    "id":"z-ai\/glm-4-32b-0414",
+    "name":"GLM 4 32B ",
+    "provider_name":"Z.ai",
+    "cost":0.1,
     "train_on_prompts":false,
-    "hf_id":"openai\/gpt-oss-120b",
-    "size":120412337472.0,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1753315200000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"z-ai\/glm-4.5-air",
+    "name":"GLM 4.5 Air",
+    "provider_name":"Z.ai",
+    "cost":0.85,
+    "train_on_prompts":false,
+    "hf_id":"zai-org\/GLM-4.5-Air",
+    "size":110468824832.0,
     "type":"open-source",
-    "license":"Apache 2.0",
-    "creation_date":1754265600000,
+    "license":"Mit",
+    "creation_date":1752969600000,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -798,16 +1974,79 @@
     ]
   },
   {
-    "id":"qwen\/qwen3-30b-a3b",
-    "name":"Qwen3 30B A3B",
-    "provider_name":"Qwen",
-    "cost":0.28,
+    "id":"z-ai\/glm-4.5v",
+    "name":"GLM 4.5V",
+    "provider_name":"Z.ai",
+    "cost":1.8,
     "train_on_prompts":false,
-    "hf_id":"Qwen\/Qwen3-30B-A3B",
-    "size":null,
+    "hf_id":"zai-org\/GLM-4.5V",
+    "size":107710933120.0,
     "type":"open-source",
-    "license":"Apache 2.0",
-    "creation_date":1745712000000,
+    "license":"Mit",
+    "creation_date":1754784000000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"z-ai\/glm-4.7-flash-20260119",
+    "name":"GLM 4.7 Flash",
+    "provider_name":"Z.ai",
+    "cost":0.4,
+    "train_on_prompts":false,
+    "hf_id":"zai-org\/GLM-4.7-Flash",
+    "size":31221488576.0,
+    "type":"open-source",
+    "license":"Mit",
+    "creation_date":1768780800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"z-ai\/glm-5-turbo-20260315",
+    "name":"GLM 5 Turbo",
+    "provider_name":"Z.ai",
+    "cost":4.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1773532800000,
+    "tasks":[
+      "translation_from",
+      "translation_to",
+      "classification",
+      "mmlu",
+      "arc",
+      "truthfulqa",
+      "mgsm"
+    ]
+  },
+  {
+    "id":"z-ai\/glm-5v-turbo-20260401",
+    "name":"GLM 5V Turbo",
+    "provider_name":"Z.ai",
+    "cost":4.0,
+    "train_on_prompts":false,
+    "hf_id":null,
+    "size":null,
+    "type":"closed-source",
+    "license":null,
+    "creation_date":1775001600000,
     "tasks":[
       "translation_from",
       "translation_to",

results/results.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75c0a0374e1e03392d76db5baf38e282b6af6dec080fab2b9cf1af7e68c931c8
-size 9437932
+oid sha256:e53e32e07c510774d3c8011f418c4bd31e36be8e6a401e6d74b1c6abbd5ad5f1
+size 11317321