Upload from GitHub Actions: blocklist: drop the grace for slow failing models, not just egregious ones

Per feedback: the 2-run grace is only worth it when retrying is cheap. main.py
now measures per-model seconds/sample during the run and flags rate-limited
models (> AUTO_BLOCKLIST_SLOW_SEC_PER_SAMPLE = 0.3s/sample); update_blocklist_
strikes excludes a failing model after ONE run if it's egregious (>=80%) OR
slow. The grace now only protects a fast, moderately-failing model — i.e. retry
only when the time investment is reasonable.

evals/main.py

@@ -9,6 +9,7 @@ from models import (
     models,
     AUTO_BLOCKLIST_MIN_ATTEMPTS,
     AUTO_BLOCKLIST_FAIL_PCT_THRESHOLD,
+    AUTO_BLOCKLIST_SLOW_SEC_PER_SAMPLE,
     update_blocklist_strikes,
     FatalAPIError,
 )
@@ -177,6 +178,10 @@ async def evaluate():
 
     results_agg = None
     budget_hit = False
+    # Models that ran slowly this run (heavily rate-limited). If they're also
+    # failing, they're excluded without a grace re-attempt — retrying an
+    # expensive failure isn't worth the time/money.
+    slow_models = set()
     for mi, model_id in enumerate(pending_models, 1):
         # Don't START a new model once over budget — stop cleanly between models.
         if over_budget():
@@ -186,9 +191,12 @@ async def evaluate():
         model_combis = combis[combis["model"] == model_id]
         print(f"[{mi}/{len(pending_models)}] {model_id}: {len(model_combis)} new samples")
         model_out = []
+        model_started = time.time()
+        attempted = 0
         for i in tqdm(range(0, len(model_combis), batch_size),
                       colour="blue", desc=model_id):
             batch = model_combis.iloc[i:i + batch_size]
+            attempted += len(batch)
             rows = [(t, m, b, s) for _, (t, m, b, s) in batch.iterrows()]
             # A single combo that throws (e.g. a flaky HF dataset download) must
             # NOT crash a multi-hour run. tqdm_asyncio.gather has no
@@ -214,6 +222,13 @@ async def evaluate():
             if over_budget():
                 budget_hit = True
                 break
+        # Flag the model as slow if it took disproportionately long per sample
+        # this run (rate-limited). Needs a meaningful sample count to be reliable.
+        sec_per_sample = (time.time() - model_started) / max(attempted, 1)
+        if attempted >= 100 and sec_per_sample > AUTO_BLOCKLIST_SLOW_SEC_PER_SAMPLE:
+            slow_models.add(model_id)
+            print(f"  ⏱ {model_id}: {sec_per_sample:.2f}s/sample (slow / rate-limited)")
+
         model_df = pd.DataFrame(model_out) if model_out else pd.DataFrame(columns=all_results.columns)
         # Persist whatever completed (full model, or partial batches if the
         # budget was hit). Partial work is saved to results-detailed so its
@@ -248,7 +263,7 @@ async def evaluate():
     # auto-blocklisted after staying broken across runs — not after a single run
     # where a provider may have rate-limited us. Non-fatal if it fails.
     try:
-        update_blocklist_strikes(all_results)
+        update_blocklist_strikes(all_results, slow_models=slow_models)
     except Exception as e:
         print(f"[main] could not update blocklist strikes: {e}")
 

evals/models.py

@@ -589,10 +589,16 @@ AUTO_BLOCKLIST_FAIL_PCT_THRESHOLD = 50.0
 AUTO_BLOCKLIST_MIN_RUNS = 2
 # Fast path: a model failing this badly is broken/unusable, not transiently
 # rate-limited — exclude it after a SINGLE bad run rather than waiting out the
-# grace period, so we don't keep spending on it. The grace only protects the
-# borderline band (FAIL_PCT_THRESHOLD .. IMMEDIATE_FAIL_PCT) where a provider
-# outage is a plausible explanation.
+# grace period, so we don't keep spending on it.
 AUTO_BLOCKLIST_IMMEDIATE_FAIL_PCT = 80.0
+# The 2-run grace is only worth it when retrying is CHEAP. A model that is
+# failing AND slow (heavily rate-limited) is expensive to retry and unlikely to
+# recover, so it's excluded immediately too. "Slow" = wall-clock seconds per
+# attempted sample above this, measured during the run. Healthy models run near
+# the 20 req/s limit (~0.05 s/sample); a rate-limited one is many times slower
+# (e.g. kimi-k2.6 ran ~0.7 s/sample). So the grace effectively only protects a
+# FAST, moderately-failing model — exactly "retry only if cheap to retry".
+AUTO_BLOCKLIST_SLOW_SEC_PER_SAMPLE = 0.3
 
 
 def compute_model_health(detailed=None) -> pd.DataFrame:
@@ -639,15 +645,20 @@ def load_auto_blocklist(date: date) -> list[str]:
     )
 
 
-def update_blocklist_strikes(detailed=None) -> pd.DataFrame:
+def update_blocklist_strikes(detailed=None, slow_models=None) -> pd.DataFrame:
     """Recompute consecutive-bad-run strikes and persist them to HF. Called once
     per eval run (from main.py) after results are merged. A model currently past
     the failure threshold gets +1 strike; a model that has recovered (or never
     failed) drops to 0 and out of the table. Models reaching AUTO_BLOCKLIST_MIN_RUNS
-    strikes are excluded by load_auto_blocklist on the NEXT run — so every model
-    gets at least one re-attempt before exclusion."""
+    strikes are excluded by load_auto_blocklist on the NEXT run.
+
+    The 2-run grace (one free re-attempt) only applies when retrying is cheap.
+    A failing model that is ALSO egregiously bad (>=80%) or SLOW (in
+    `slow_models`, measured this run) is excluded after a single run — we don't
+    spend more time/money re-attempting an expensive failure."""
     from datasets_.util import load, save
 
+    slow_models = slow_models or set()
     health = compute_model_health(detailed)
     cols = ["model", "strikes", "failed_pct"]
     if health.empty:
@@ -668,11 +679,12 @@ def update_blocklist_strikes(detailed=None) -> pd.DataFrame:
     fail_map = dict(zip(bad["model"], bad["failed_pct"]))
 
     def _strikes_for(model, fail_pct):
-        # Egregious failure → jump straight to the exclusion threshold (no
-        # grace). Otherwise increment so the borderline band needs MIN_RUNS
-        # consecutive bad runs.
+        # No grace if the failure is egregious OR slow-and-expensive-to-retry;
+        # otherwise increment so a fast, moderately-failing model gets one free
+        # re-attempt before exclusion.
         incremented = int(prior_map.get(model, 0)) + 1
-        if fail_pct >= AUTO_BLOCKLIST_IMMEDIATE_FAIL_PCT:
+        no_grace = fail_pct >= AUTO_BLOCKLIST_IMMEDIATE_FAIL_PCT or model in slow_models
+        if no_grace:
             return max(incremented, AUTO_BLOCKLIST_MIN_RUNS)
         return incremented