update submission

app.py

@@ -536,6 +536,144 @@ def create_capability_subplots(data_dict, title="Capability Performance", top_n=
     return fig
 
 
+def _rank_prefix(i):
+    medals = ["🥇", "🥈", "🥉"]
+    return f"{medals[i]} {i+1}" if i < 3 else str(i + 1)
+
+
+def _fmt(v):
+    return f"{v * 100:.2f}%"
+
+
+def _build_rows_sorted(items, verified_dict, score_fn, type_name):
+    """
+    Build rows for verified entries only (verified=True).
+    Unverified submissions are excluded from the leaderboard display.
+    """
+    rows = []
+    for item in sorted(items):
+        if not verified_dict.get(item, False):
+            continue
+        row = score_fn(item, True, type_name)
+        rows.append(row)
+
+    rows.sort(key=lambda r: r["_sort"], reverse=True)
+    for i, r in enumerate(rows):
+        r["Rank"] = _rank_prefix(i)
+
+    return rows
+
+
+def create_capability_table(capability_dict, domain_dict, verified_dict, type_name="Agent"):
+    """
+    Summary table grouped by capability (A/B/C/D).
+    verified=True  → ranked by official score
+    verified=False → appended unranked, scores marked with * (self-reported)
+    """
+    items = set()
+    for d in domain_dict.values():
+        items.update(d.keys())
+    if not items:
+        return pd.DataFrame()
+
+    cap_cols = {
+        "Recall":            "Recall (A)",
+        "Causal Inference":  "Causal Inf. (B)",
+        "State Updating":    "State Upd. (C)",
+        "State Abstraction": "State Abs. (D)",
+    }
+    cap_weights = {}
+    if QA_DISTRIBUTION:
+        pt = QA_DISTRIBUTION.get("overall_distribution", {}).get("problem_types", {})
+        letter_to_cap = {"A": "Recall", "B": "Causal Inference",
+                         "C": "State Updating", "D": "State Abstraction"}
+        for letter, info in pt.items():
+            cap_weights[letter_to_cap.get(letter, "")] = info.get("ratio", 0.0)
+
+    def score_fn(item, is_verified, type_name):
+        model_family = ""
+        for cd in capability_dict.values():
+            if item in cd and isinstance(cd[item], dict):
+                model_family = cd[item].get("model_family", "")
+                if model_family:
+                    break
+
+        cap_scores = {}
+        for cap_name in cap_cols:
+            d = capability_dict.get(cap_name, {}).get(item, {})
+            cap_scores[cap_name] = d.get("accuracy", 0.0) if isinstance(d, dict) else 0.0
+
+        w_sum = sum(cap_scores[c] * cap_weights.get(c, 0.0) for c in cap_cols)
+        w_tot = sum(cap_weights.get(c, 0.0) for c in cap_cols)
+        avg = w_sum / w_tot if w_tot > 0 else sum(cap_scores.values()) / len(cap_scores)
+
+        row = {
+            type_name:      f"{item} {'✓' if is_verified else '○'}",
+            "Model Family": model_family,
+            "Avg Score":    _fmt(avg),
+            "_sort":        avg,
+        }
+        for cap_name, col_label in cap_cols.items():
+            row[col_label] = _fmt(cap_scores[cap_name])
+        return row
+
+    rows = _build_rows_sorted(items, verified_dict, score_fn, type_name)
+    return pd.DataFrame([
+        {"Rank": r["Rank"], **{k: v for k, v in r.items() if k not in ("Rank", "_sort")}}
+        for r in rows
+    ])
+
+
+def create_domain_table(capability_dict, domain_dict, verified_dict, type_name="Agent"):
+    """
+    Summary table grouped by domain.
+    verified=True  → ranked by official score
+    verified=False → appended unranked, scores marked with * (self-reported)
+    """
+    items = set()
+    for d in domain_dict.values():
+        items.update(d.keys())
+    if not items:
+        return pd.DataFrame()
+
+    domain_order = ["TEXT2SQL", "SOFTWARE", "WEB", "GAME", "EMBODIED_AI", "OPENWORLD_QA"]
+    domain_weights = {}
+    if QA_DISTRIBUTION:
+        for dom, info in QA_DISTRIBUTION.get("domain_distribution", {}).items():
+            domain_weights[dom] = info.get("qa_ratio", 0.0)
+
+    def score_fn(item, is_verified, type_name):
+        model_family = ""
+        for cd in capability_dict.values():
+            if item in cd and isinstance(cd[item], dict):
+                model_family = cd[item].get("model_family", "")
+                if model_family:
+                    break
+
+        dom_scores = {}
+        for dom in domain_order:
+            d = domain_dict.get(dom, {}).get(item, {})
+            dom_scores[dom] = d.get("accuracy", 0.0) if isinstance(d, dict) else 0.0
+
+        w_sum = sum(dom_scores[d] * domain_weights.get(d, 0.0) for d in domain_order)
+        w_tot = sum(domain_weights.get(d, 0.0) for d in domain_order)
+        avg = w_sum / w_tot if w_tot > 0 else sum(dom_scores.values()) / len(dom_scores)
+
+        row = {
+            type_name:      f"{item} {'✓' if is_verified else '○'}",
+            "Model Family": model_family,
+            "Avg Score":    _fmt(avg),
+            "_sort":        avg,
+        }
+        for dom in domain_order:
+            row[dom] = _fmt(dom_scores[dom])
+        return row
+
+    rows = _build_rows_sorted(items, verified_dict, score_fn, type_name)
+    return pd.DataFrame([{"Rank": r["Rank"], **{k: v for k, v in r.items() if k != "Rank" and k != "_sort"}}
+                          for r in rows])
+
+
 def create_summary_table(capability_dict, domain_dict, verified_dict, type_name="Agent"):
     """
     Create summary table showing rank, average accuracy and F1 scores.
@@ -675,19 +813,38 @@ def build_app():
     if not any(len(category_data) > 0 for category_data in model_domain_filtered.values()):
         model_domain_filtered = {}
 
-    with gr.Blocks(css=CSS, title="AMA-Bench Leaderboard", theme=gr.themes.Soft()) as demo:
+    import base64, pathlib
+    _logo_path = pathlib.Path("assets/ama_logo.jpg")
+    if _logo_path.exists():
+        _logo_b64 = base64.b64encode(_logo_path.read_bytes()).decode()
+        _logo_tag = (
+            '<img src="data:image/jpeg;base64,' + _logo_b64 + '"'
+            ' alt="AMA-Bench" style="height:80px;object-fit:contain;flex-shrink:0;">'
+        )
+    else:
+        _logo_tag = "🤖 "
+
+    with gr.Blocks(title="AMA-Bench Leaderboard") as demo:
 
         # Header
-        gr.HTML("""
-        <div style="text-align: center; padding: 10px 20px; margin-bottom: 20px;">
-            <h1 style="margin: 0; font-size: 48px; font-weight: 700; color: #1a1a2e;">
-                🤖 AMA-Bench: Leaderboard
-            </h1>
-            <p style="font-size: 18px; color: #666; margin-top: 10px;">
-                Agent Memory Assessment Benchmark - Performance Visualization
-            </p>
+        gr.HTML(
+            """
+        <div style="display:flex; align-items:center; justify-content:center;
+                    gap:24px; padding:20px 20px 10px; margin-bottom:20px;">
+        """
+            + _logo_tag
+            + """
+            <div style="text-align:left;">
+                <h1 style="margin:0; font-size:48px; font-weight:700; color:#1a1a2e; line-height:1.1;">
+                    AMA-Bench: Leaderboard
+                </h1>
+                <p style="font-size:18px; color:#666; margin:8px 0 0;">
+                    Agent Memory Assessment Benchmark - Performance Visualization
+                </p>
+            </div>
         </div>
-        """)
+        """
+        )
 
         # Welcome Banner
         gr.HTML("""
@@ -749,13 +906,11 @@ def build_app():
 
                         with gr.Accordion("📊 Summary Statistics", open=True):
                             gr.Markdown("""
-                            **Verification Status:** Icons next to agent names indicate verification status
-                            • **✓** = Officially verified by LLM-as-Judge
-                            • **○** = User-submitted, pending official verification
+                            **Verification Status:** Only officially verified entries (✓) are shown. User-submitted results (○) will appear after weekly LLM-as-Judge evaluation.
                             """)
                             agent_domain_table = gr.Dataframe(
-                                value=create_summary_table(AGENT_CAPABILITY, AGENT_DOMAIN, AGENT_VERIFIED, "Agent"),
-                                label="Average Domain Scores"
+                                value=create_domain_table(AGENT_CAPABILITY, AGENT_DOMAIN, AGENT_VERIFIED, "Agent"),
+                                label="Scores by Domain"
                             )
 
                         # Update chart when slider changes
@@ -796,13 +951,11 @@ def build_app():
 
                         with gr.Accordion("📊 Summary Statistics", open=True):
                             gr.Markdown("""
-                            **Verification Status:** Icons next to agent names indicate verification status
-                            • **✓** = Officially verified by LLM-as-Judge
-                            • **○** = User-submitted, pending official verification
+                            **Verification Status:** Only officially verified entries (✓) are shown. User-submitted results (○) will appear after weekly LLM-as-Judge evaluation.
                             """)
                             agent_capability_table = gr.Dataframe(
-                                value=create_summary_table(AGENT_CAPABILITY, AGENT_DOMAIN, AGENT_VERIFIED, "Agent"),
-                                label="Average Capability Scores"
+                                value=create_capability_table(AGENT_CAPABILITY, AGENT_DOMAIN, AGENT_VERIFIED, "Agent"),
+                                label="Scores by Capability"
                             )
 
                         # Update chart when slider changes
@@ -853,13 +1006,11 @@ def build_app():
 
                         with gr.Accordion("📊 Summary Statistics", open=True):
                             gr.Markdown("""
-                            **Verification Status:** Icons next to model names indicate verification status
-                            • **✓** = Officially verified by LLM-as-Judge
-                            • **○** = User-submitted, pending official verification
+                            **Verification Status:** Only officially verified entries (✓) are shown. User-submitted results (○) will appear after weekly LLM-as-Judge evaluation.
                             """)
                             model_domain_table = gr.Dataframe(
-                                value=create_summary_table(MODEL_CAPABILITY, model_domain_filtered, MODEL_VERIFIED, "Model"),
-                                label="Average Domain Scores"
+                                value=create_domain_table(MODEL_CAPABILITY, model_domain_filtered, MODEL_VERIFIED, "Model"),
+                                label="Scores by Domain"
                             )
 
                         # Update chart when slider changes
@@ -900,13 +1051,11 @@ def build_app():
 
                         with gr.Accordion("📊 Summary Statistics", open=True):
                             gr.Markdown("""
-                            **Verification Status:** Icons next to model names indicate verification status
-                            • **✓** = Officially verified by LLM-as-Judge
-                            • **○** = User-submitted, pending official verification
+                            **Verification Status:** Only officially verified entries (✓) are shown. User-submitted results (○) will appear after weekly LLM-as-Judge evaluation.
                             """)
                             model_capability_table = gr.Dataframe(
-                                value=create_summary_table(MODEL_CAPABILITY, MODEL_DOMAIN, MODEL_VERIFIED, "Model"),
-                                label="Average Capability Scores"
+                                value=create_capability_table(MODEL_CAPABILITY, MODEL_DOMAIN, MODEL_VERIFIED, "Model"),
+                                label="Scores by Capability"
                             )
 
                         # Update chart when slider changes
@@ -971,48 +1120,56 @@ def build_app():
                         )
 
                 gr.Markdown("""
-                **📋 New Submission Format:**
+                **📋 Submission Format:**
 
-                Your JSONL file should contain one submission per episode with the following fields:
+                Your JSONL file should contain one line per episode:
 
                 ```json
                 {
                   "episode_id": "trajectory_id",
-                  "answer_list": ["xxxx", "xxxx", "xxx"],
+                  "question_uuid_list": ["uuid-1", "uuid-2", "uuid-3"],
+                  "answer_list": ["The agent moved right.", "..."],
                   "llm_as_judge_score_list": [true, false, true]
                 }
                 ```
 
                 **Field Descriptions:**
-                - `episode_id` (required): The episode identifier
-                - `answer_list` (required): List of your model/agent's answers (one per question in the episode)
-                - `llm_as_judge_score_list` (optional): List of boolean values (true/false) indicating correctness
-                  - If provided, these are your self-evaluated scores
-                  - If not provided, scoring will be done during weekly official evaluation
-                  - Official scores (`verified=true`) are computed by our LLM-as-Judge system
+                - `episode_id` *(required)*: The episode identifier — used to automatically look up the domain
+                - `question_uuid_list` *(required)*: UUIDs of the benchmark questions in the same order as `answer_list` — used to look up each question's capability (A/B/C/D).
+                - `answer_list` *(required)*: Your model/agent's answers, one per question
+                - `llm_as_judge_score_list` *(required)*: `true`/`false` per answer — your self-evaluated correctness scores used for leaderboard ranking.
 
                 **Important Notes:**
-                - `answer_list` and `llm_as_judge_score_list` must have the same length
-                - All submissions start as `verified=false` and become `verified=true` after official evaluation
+                - `question_uuid_list`, `answer_list`, and `llm_as_judge_score_list` must all be the same length
+                - Domain is resolved automatically from `episode_id`; capability (A/B/C/D) is resolved from `question_uuid_list` — no need to supply them manually
+                - All submissions start as `verified=false` and become `verified=true` after official LLM-as-Judge evaluation
                 """)
 
                 with gr.Row():
                     submit_button = gr.Button("Submit", variant="primary", size="lg")
 
-                submission_result = gr.Markdown()
+                submission_result = gr.HTML()
 
                 submit_button.click(
-                    add_new_submission,
-                    [
+                    fn=lambda: gr.update(interactive=False, value="⏳ Submitting..."),
+                    inputs=[],
+                    outputs=[submit_button],
+                ).then(
+                    fn=add_new_submission,
+                    inputs=[
                         model_name_textbox,
                         submission_type,
                         url_textbox,
                         file_upload,
                         organisation,
                         mail,
-                        model_family_textbox
+                        model_family_textbox,
                     ],
-                    submission_result,
+                    outputs=[submission_result],
+                ).then(
+                    fn=lambda: gr.update(interactive=True, value="Submit"),
+                    inputs=[],
+                    outputs=[submit_button],
                 )
 
             # ============================================================
@@ -1078,4 +1235,4 @@ Results are reported as **Accuracy** and **F1 Score**:
 
 if __name__ == "__main__":
     demo_app = build_app()
-    demo_app.launch(debug=True, show_error=True)
+    demo_app.launch(debug=True, show_error=True, theme=gr.themes.Soft())

content.py

@@ -16,12 +16,14 @@ Results can be submitted for evaluation. Each submission should contain answers
 
 We expect submissions to be JSON Lines files with the following format:
 ```
-{"episode_id": "traj_id_1", "answer_list": ["(A)", "(B)(C)", "(D)"], "reasoning_trace": "optional"}
+{"episode_id": "trajectory_id", "question_uuid_list": ["uuid-1", "uuid-2", "uuid-3"], "answer_list": ["The agent moved right.", "..."], "llm_as_judge_score_list": [true, false, true]}
 ```
 
 **Required fields:**
 - `episode_id`: The episode identifier
-- `answer_list`: Your model's answer list for the questions in the episode (a list of strings, e.g., ["(A)", "(B)(C)", "(D)"])
+- `question_uuid_list`: List of question UUIDs corresponding to each answer (e.g., `["uuid-1", "uuid-2"]`)
+- `answer_list`: Your model's answers, in the same order as `question_uuid_list`
+- `llm_as_judge_score_list`: Boolean scores for each answer (e.g., `[true, false, true]`)
 - `reasoning_trace`: (Optional) The reasoning process or explanation for the answers
 """
 
@@ -52,5 +54,4 @@ def model_hyperlink(link, model_name):
     """Create a hyperlink to the model information."""
     if not link or link.strip() == "":
         return model_name
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'

data/model.jsonl

@@ -1,8 +1,36 @@
-{"model": "Claude Haiku 3.5", "Date": "2026-03-03", "verified": true, "Score": {"GAME": [{"A": 0.5}, {"B": 0.458}, {"C": 0.564}, {"D": 0.583}], "EMBODIED_AI": [{"A": 0.3934}, {"B": 0.4667}, {"C": 0.34}, {"D": 0.0}], "WEB": [{"A": 0.4711}, {"B": 0.5889}, {"C": 0.5222}, {"D": 0.5932}], "TEXT2SQL": [{"A": 0.6233}, {"B": 0.1961}, {"C": 0.4328}, {"D": 0.1569}], "OPENWORLD_QA": [{"A": 0.6596}, {"B": 0.7333}, {"C": 0.5625}, {"D": 0.5}], "SOFTWARE": [{"A": 0.26}, {"B": 0.4366}, {"C": 0.1739}, {"D": 0.1324}]}}
-{"model": "GPT-5 mini", "Date": "2026-03-03", "verified": true, "Score": {"GAME": [{"A": 0.5}, {"B": 0.514}, {"C": 0.4872}, {"D": 0.5}], "EMBODIED_AI": [{"A": 0.6557}, {"B": 0.5667}, {"C": 0.4133}, {"D": 0.0169}], "WEB": [{"A": 0.8}, {"B": 0.8925}, {"C": 0.8387}, {"D": 0.7869}], "TEXT2SQL": [{"A": 0.8924}, {"B": 0.7778}, {"C": 0.8731}, {"D": 0.7941}], "OPENWORLD_QA": [{"A": 0.7347}, {"B": 0.8105}, {"C": 0.757}, {"D": 0.85}], "SOFTWARE": [{"A": 0.4811}, {"B": 0.6933}, {"C": 0.4521}, {"D": 0.6667}]}}
-{"model": "gpt 5.2", "Date": "2026-03-03", "verified": true, "Score": {"GAME": [{"A": 0.8362}, {"B": 0.8194}, {"C": 0.7564}, {"D": 0.8333}], "EMBODIED_AI": [{"A": 0.9508}, {"B": 0.8}, {"C": 0.5067}, {"D": 0.0}], "WEB": [{"A": 0.744}, {"B": 0.8925}, {"C": 0.7204}, {"D": 0.7705}], "TEXT2SQL": [{"A": 0.9058}, {"B": 0.8627}, {"C": 0.8806}, {"D": 0.6765}], "OPENWORLD_QA": [{"A": 0.6939}, {"B": 0.7158}, {"C": 0.5794}, {"D": 0.65}], "SOFTWARE": [{"A": 0.4623}, {"B": 0.7067}, {"C": 0.3699}, {"D": 0.625}]}}
-{"model": "Gemini 2.5 flash", "Date": "2026-03-03", "verified": true, "Score": {"GAME": [{"A": 0.4224}, {"B": 0.1667}, {"C": 0.1795}, {"D": 0.1333}], "EMBODIED_AI": [{"A": 0.541}, {"B": 0.6333}, {"C": 0.4}, {"D": 0.0169}], "WEB": [{"A": 0.664}, {"B": 0.6237}, {"C": 0.6344}, {"D": 0.4918}], "TEXT2SQL": [{"A": 0.688}, {"B": 0.6344}, {"C": 0.6559}, {"D": 0.5246}], "OPENWORLD_QA": [{"A": 0.77}, {"B": 0.619}, {"C": 0.721}, {"D": 0.758}], "SOFTWARE": [{"A": 0.3726}, {"B": 0.32}, {"C": 0.3288}, {"D": 0.5139}]}}
-{"model": "Qwen2.5-14B-Instruct-1M", "Date": "2026-03-03", "verified": true, "Score": {"GAME": [{"A": 0.4957}, {"B": 0.4444}, {"C": 0.5769}, {"D": 0.4833}], "EMBODIED_AI": [{"A": 0.6842}, {"B": 0.4211}, {"C": 0.2211}, {"D": 0.0}], "WEB": [{"A": 0.488}, {"B": 0.6882}, {"C": 0.5376}, {"D": 0.4918}], "TEXT2SQL": [{"A": 0.6319}, {"B": 0.2339}, {"C": 0.5526}, {"D": 0.119}], "OPENWORLD_QA": [{"A": 0.48}, {"B": 0.4091}, {"C": 0.6087}, {"D": 0.5}], "SOFTWARE": [{"A": 0.5189}, {"B": 0.3867}, {"C": 0.3014}, {"D": 0.5417}]}}
-{"model": "Qwen3-32B", "Date": "2026-03-03", "verified": true, "Score": {"GAME": [{"A": 0.5431}, {"B": 0.4861}, {"C": 0.5128}, {"D": 0.6}], "EMBODIED_AI": [{"A": 0.7966}, {"B": 0.6437}, {"C": 0.4345}, {"D": 0.0526}], "WEB": [{"A": 0.504}, {"B": 0.6667}, {"C": 0.5054}, {"D": 0.541}], "TEXT2SQL": [{"A": 0.7309}, {"B": 0.3203}, {"C": 0.5672}, {"D": 0.2059}], "OPENWORLD_QA": [{"A": 0.4894}, {"B": 0.5978}, {"C": 0.4904}, {"D": 0.4138}], "SOFTWARE": [{"A": 0.5581}, {"B": 0.5254}, {"C": 0.3898}, {"D": 0.4655}]}}
-{"model": "Qwen3-14B", "Date": "2026-03-03", "verified": true, "Score": {"GAME": [{"A": 0.5162}, {"B": 0.5}, {"C": 0.5556}, {"D": 0.6}], "EMBODIED_AI": [{"A": 0.6557}, {"B": 0.5}, {"C": 0.3133}, {"D": 0.0508}], "WEB": [{"A": 0.512}, {"B": 0.5591}, {"C": 0.5484}, {"D": 0.5246}], "TEXT2SQL": [{"A": 0.6502}, {"B": 0.2288}, {"C": 0.4851}, {"D": 0.1471}], "OPENWORLD_QA": [{"A": 0.5306}, {"B": 0.5895}, {"C": 0.4393}, {"D": 0.3667}], "SOFTWARE": [{"A": 0.4953}, {"B": 0.4267}, {"C": 0.3425}, {"D": 0.3194}]}}
-{"model": "Qwen3-8B", "Date": "2026-03-03", "verified": true, "Score": {"GAME": [{"A": 0.5294}, {"B": 0.4111}, {"C": 0.4333}, {"D": 0.5}], "EMBODIED_AI": [{"A": 0.541}, {"B": 0.3333}, {"C": 0.2867}, {"D": 0.0169}], "WEB": [{"A": 0.424}, {"B": 0.4839}, {"C": 0.4409}, {"D": 0.3934}], "TEXT2SQL": [{"A": 0.5605}, {"B": 0.1895}, {"C": 0.4478}, {"D": 0.1471}], "OPENWORLD_QA": [{"A": 0.4796}, {"B": 0.5263}, {"C": 0.4579}, {"D": 0.4333}], "SOFTWARE": [{"A": 0.4481}, {"B": 0.44}, {"C": 0.3014}, {"D": 0.3472}]}}
+{"model": "Claude Haiku 3.5", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.5}, {"B": 0.458}, {"C": 0.564}, {"D": 0.583}], "EMBODIED_AI": [{"A": 0.3934}, {"B": 0.4667}, {"C": 0.34}, {"D": 0.0}], "WEB": [{"A": 0.4711}, {"B": 0.5889}, {"C": 0.5222}, {"D": 0.5932}], "TEXT2SQL": [{"A": 0.6233}, {"B": 0.1961}, {"C": 0.4328}, {"D": 0.1569}], "OPENWORLD_QA": [{"A": 0.6596}, {"B": 0.7333}, {"C": 0.5625}, {"D": 0.5}], "SOFTWARE": [{"A": 0.26}, {"B": 0.4366}, {"C": 0.1739}, {"D": 0.1324}]}}
+{"model": "OpenAI GPT-5.1 mini", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.5}, {"B": 0.514}, {"C": 0.4872}, {"D": 0.5}], "EMBODIED_AI": [{"A": 0.6557}, {"B": 0.5667}, {"C": 0.4133}, {"D": 0.0169}], "WEB": [{"A": 0.8}, {"B": 0.8925}, {"C": 0.8387}, {"D": 0.7869}], "TEXT2SQL": [{"A": 0.8924}, {"B": 0.7778}, {"C": 0.8731}, {"D": 0.7941}], "OPENWORLD_QA": [{"A": 0.7347}, {"B": 0.8105}, {"C": 0.757}, {"D": 0.85}], "SOFTWARE": [{"A": 0.4811}, {"B": 0.6933}, {"C": 0.4521}, {"D": 0.6667}]}}
+{"model": "gpt 5.2", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.8362}, {"B": 0.8194}, {"C": 0.7564}, {"D": 0.8333}], "EMBODIED_AI": [{"A": 0.9508}, {"B": 0.8}, {"C": 0.5067}, {"D": 0.0}], "WEB": [{"A": 0.744}, {"B": 0.8925}, {"C": 0.7204}, {"D": 0.7705}], "TEXT2SQL": [{"A": 0.9058}, {"B": 0.8627}, {"C": 0.8806}, {"D": 0.6765}], "OPENWORLD_QA": [{"A": 0.6939}, {"B": 0.7158}, {"C": 0.5794}, {"D": 0.65}], "SOFTWARE": [{"A": 0.4623}, {"B": 0.7067}, {"C": 0.3699}, {"D": 0.625}]}}
+{"model": "Gemini 2.5 flash", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.4224}, {"B": 0.1667}, {"C": 0.1795}, {"D": 0.1333}], "EMBODIED_AI": [{"A": 0.541}, {"B": 0.6333}, {"C": 0.4}, {"D": 0.0169}], "WEB": [{"A": 0.664}, {"B": 0.6237}, {"C": 0.6344}, {"D": 0.4918}], "TEXT2SQL": [{"A": 0.688}, {"B": 0.6344}, {"C": 0.6559}, {"D": 0.5246}], "OPENWORLD_QA": [{"A": 0.77}, {"B": 0.619}, {"C": 0.721}, {"D": 0.758}], "SOFTWARE": [{"A": 0.3726}, {"B": 0.32}, {"C": 0.3288}, {"D": 0.5139}]}}
+{"model": "Qwen2.5-14B-Instruct-1M", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.4957}, {"B": 0.4444}, {"C": 0.5769}, {"D": 0.4833}], "EMBODIED_AI": [{"A": 0.6842}, {"B": 0.4211}, {"C": 0.2211}, {"D": 0.0}], "WEB": [{"A": 0.488}, {"B": 0.6882}, {"C": 0.5376}, {"D": 0.4918}], "TEXT2SQL": [{"A": 0.6319}, {"B": 0.2339}, {"C": 0.5526}, {"D": 0.119}], "OPENWORLD_QA": [{"A": 0.48}, {"B": 0.4091}, {"C": 0.6087}, {"D": 0.5}], "SOFTWARE": [{"A": 0.5189}, {"B": 0.3867}, {"C": 0.3014}, {"D": 0.5417}]}}
+{"model": "Qwen3-32B", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.5431}, {"B": 0.4861}, {"C": 0.5128}, {"D": 0.6}], "EMBODIED_AI": [{"A": 0.7966}, {"B": 0.6437}, {"C": 0.4345}, {"D": 0.0526}], "WEB": [{"A": 0.504}, {"B": 0.6667}, {"C": 0.5054}, {"D": 0.541}], "TEXT2SQL": [{"A": 0.7309}, {"B": 0.3203}, {"C": 0.5672}, {"D": 0.2059}], "OPENWORLD_QA": [{"A": 0.4894}, {"B": 0.5978}, {"C": 0.4904}, {"D": 0.4138}], "SOFTWARE": [{"A": 0.5581}, {"B": 0.5254}, {"C": 0.3898}, {"D": 0.4655}]}}
+{"model": "Qwen3-14B", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.5162}, {"B": 0.5}, {"C": 0.5556}, {"D": 0.6}], "EMBODIED_AI": [{"A": 0.6557}, {"B": 0.5}, {"C": 0.3133}, {"D": 0.0508}], "WEB": [{"A": 0.512}, {"B": 0.5591}, {"C": 0.5484}, {"D": 0.5246}], "TEXT2SQL": [{"A": 0.6502}, {"B": 0.2288}, {"C": 0.4851}, {"D": 0.1471}], "OPENWORLD_QA": [{"A": 0.5306}, {"B": 0.5895}, {"C": 0.4393}, {"D": 0.3667}], "SOFTWARE": [{"A": 0.4953}, {"B": 0.4267}, {"C": 0.3425}, {"D": 0.3194}]}}
+{"model": "Qwen3-8B", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.5294}, {"B": 0.4111}, {"C": 0.4333}, {"D": 0.5}], "EMBODIED_AI": [{"A": 0.541}, {"B": 0.3333}, {"C": 0.2867}, {"D": 0.0169}], "WEB": [{"A": 0.424}, {"B": 0.4839}, {"C": 0.4409}, {"D": 0.3934}], "TEXT2SQL": [{"A": 0.5605}, {"B": 0.1895}, {"C": 0.4478}, {"D": 0.1471}], "OPENWORLD_QA": [{"A": 0.4796}, {"B": 0.5263}, {"C": 0.4579}, {"D": 0.4333}], "SOFTWARE": [{"A": 0.4481}, {"B": 0.44}, {"C": 0.3014}, {"D": 0.3472}]}}
+{"model": "BM25 (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.3365}, {"B": 0.3333}, {"C": 0.4638}, {"D": 0.2963}], "EMBODIED_AI": [{"A": 0.2295}, {"B": 0.3111}, {"C": 0.0933}, {"D": 0.0008}], "WEB": [{"A": 0.272}, {"B": 0.3548}, {"C": 0.2366}, {"D": 0.1311}], "TEXT2SQL": [{"A": 0.3857}, {"B": 0.4183}, {"C": 0.3582}, {"D": 0.1078}], "OPENWORLD_QA": [{"A": 0.2347}, {"B": 0.3474}, {"C": 0.3178}, {"D": 0.2167}], "SOFTWARE": [{"A": 0.467}, {"B": 0.76}, {"C": 0.5616}, {"D": 0.7778}]}}
+{"model": "Qwen3-Embedding-4B (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.4914}, {"B": 0.5278}, {"C": 0.5769}, {"D": 0.4667}], "EMBODIED_AI": [{"A": 0.4262}, {"B": 0.2444}, {"C": 0.1467}, {"D": 0.0}], "WEB": [{"A": 0.312}, {"B": 0.4194}, {"C": 0.2043}, {"D": 0.2131}], "TEXT2SQL": [{"A": 0.5785}, {"B": 0.5556}, {"C": 0.306}, {"D": 0.2255}], "OPENWORLD_QA": [{"A": 0.4388}, {"B": 0.4632}, {"C": 0.4112}, {"D": 0.2833}], "SOFTWARE": [{"A": 0.5849}, {"B": 0.7067}, {"C": 0.4795}, {"D": 0.625}]}}
+{"model": "GRAPHRAG (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.5347}, {"B": 0.5484}, {"C": 0.5588}, {"D": 0.5962}], "EMBODIED_AI": [{"A": 0.0426}, {"B": 0.3913}, {"C": 0.1565}, {"D": 0.0}], "WEB": [{"A": 0.314}, {"B": 0.4778}, {"C": 0.4333}, {"D": 0.4576}], "TEXT2SQL": [{"A": 0.2646}, {"B": 0.2353}, {"C": 0.2687}, {"D": 0.098}], "OPENWORLD_QA": [{"A": 0.3474}, {"B": 0.4348}, {"C": 0.233}, {"D": 0.2586}], "SOFTWARE": [{"A": 0.3585}, {"B": 0.36}, {"C": 0.2603}, {"D": 0.4167}]}}
+{"model": "Hipporag2 (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.5137}, {"B": 0.6029}, {"C": 0.6806}, {"D": 0.625}], "EMBODIED_AI": [{"A": 0.3607}, {"B": 0.2111}, {"C": 0.1333}, {"D": 0.0}], "WEB": [{"A": 0.344}, {"B": 0.4731}, {"C": 0.4086}, {"D": 0.2787}], "TEXT2SQL": [{"A": 0.6233}, {"B": 0.5882}, {"C": 0.4627}, {"D": 0.1765}], "OPENWORLD_QA": [{"A": 0.4595}, {"B": 0.5135}, {"C": 0.5122}, {"D": 0.3478}], "SOFTWARE": [{"A": 0.3471}, {"B": 0.5902}, {"C": 0.4237}, {"D": 0.7586}]}}
+{"model": "Memagent (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.3103}, {"B": 0.3056}, {"C": 0.3718}, {"D": 0.2833}], "EMBODIED_AI": [{"A": 0.0656}, {"B": 0.2889}, {"C": 0.0533}, {"D": 0.0169}], "WEB": [{"A": 0.256}, {"B": 0.3118}, {"C": 0.2258}, {"D": 0.2623}], "TEXT2SQL": [{"A": 0.2518}, {"B": 0.2975}, {"C": 0.272}, {"D": 0.1602}], "OPENWORLD_QA": [{"A": 0.1939}, {"B": 0.1895}, {"C": 0.1495}, {"D": 0.1}], "SOFTWARE": [{"A": 0.4292}, {"B": 0.6267}, {"C": 0.6027}, {"D": 0.5}]}}
+{"model": "Mem1 (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.2217}, {"B": 0.24}, {"C": 0.1918}, {"D": 0.25}], "EMBODIED_AI": [{"A": 0.0164}, {"B": 0.1111}, {"C": 0.0067}, {"D": 0.0}], "WEB": [{"A": 0.12}, {"B": 0.1075}, {"C": 0.1828}, {"D": 0.1148}], "TEXT2SQL": [{"A": 0.0762}, {"B": 0.0784}, {"C": 0.0746}, {"D": 0.0294}], "OPENWORLD_QA": [{"A": 0.1224}, {"B": 0.2}, {"C": 0.0935}, {"D": 0.0667}], "SOFTWARE": [{"A": 0.1698}, {"B": 0.16}, {"C": 0.1918}, {"D": 0.2222}]}}
+{"model": "Amem (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.3793}, {"B": 0.4028}, {"C": 0.5}, {"D": 0.4167}], "EMBODIED_AI": [{"A": 0.082}, {"B": 0.4}, {"C": 0.22}, {"D": 0.0339}], "WEB": [{"A": 0.36}, {"B": 0.4624}, {"C": 0.3011}, {"D": 0.4426}], "TEXT2SQL": [{"A": 0.4036}, {"B": 0.3333}, {"C": 0.3134}, {"D": 0.2059}], "OPENWORLD_QA": [{"A": 0.2755}, {"B": 0.3579}, {"C": 0.2243}, {"D": 0.3167}], "SOFTWARE": [{"A": 0.283}, {"B": 0.2667}, {"C": 0.2877}, {"D": 0.3472}]}}
+{"model": "Mem0 (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.4259}, {"B": 0.3788}, {"C": 0.473}, {"D": 0.2857}], "EMBODIED_AI": [{"A": 0.0164}, {"B": 0.3444}, {"C": 0.0667}, {"D": 0.0169}], "WEB": [{"A": 0.256}, {"B": 0.3656}, {"C": 0.2473}, {"D": 0.2131}], "TEXT2SQL": [{"A": 0.16}, {"B": 0.1512}, {"C": 0.1139}, {"D": 0.0517}], "OPENWORLD_QA": [{"A": 0.1735}, {"B": 0.1895}, {"C": 0.1682}, {"D": 0.1167}], "SOFTWARE": [{"A": 0.1953}, {"B": 0.234}, {"C": 0.2444}, {"D": 0.2727}]}}
+{"model": "Memorag (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.5405}, {"B": 0.6}, {"C": 0.64}, {"D": 0.45}], "EMBODIED_AI": [{"A": 0.1639}, {"B": 0.1111}, {"C": 0.0667}, {"D": 0.0}], "WEB": [{"A": 0.368}, {"B": 0.4839}, {"C": 0.2473}, {"D": 0.3607}], "TEXT2SQL": [{"A": 0.7273}, {"B": 0.7333}, {"C": 0.6154}, {"D": 0.4}], "OPENWORLD_QA": [{"A": 0.4}, {"B": 0.5789}, {"C": 0.3333}, {"D": 0.3333}], "SOFTWARE": [{"A": 0.4495}, {"B": 0.6494}, {"C": 0.5067}, {"D": 0.5946}]}}
+{"model": "Memgpt (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.3534}, {"B": 0.3472}, {"C": 0.4744}, {"D": 0.5667}], "EMBODIED_AI": [{"A": 0.1475}, {"B": 0.2444}, {"C": 0.04}, {"D": 0.0169}], "WEB": [{"A": 0.384}, {"B": 0.4624}, {"C": 0.2688}, {"D": 0.1967}], "TEXT2SQL": [{"A": 0.287}, {"B": 0.3595}, {"C": 0.1418}, {"D": 0.0392}], "OPENWORLD_QA": [{"A": 0.2615}, {"B": 0.5156}, {"C": 0.3099}, {"D": 0.175}], "SOFTWARE": [{"A": 0.5385}, {"B": 0.7333}, {"C": 0.5111}, {"D": 0.6136}]}}
+{"model": "Mem-alpha (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.4286}, {"B": 0.4783}, {"C": 0.4868}, {"D": 0.3621}], "EMBODIED_AI": [{"A": 0.0984}, {"B": 0.4222}, {"C": 0.1}, {"D": 0.0}], "WEB": [{"A": 0.3077}, {"B": 0.5172}, {"C": 0.2759}, {"D": 0.3509}], "TEXT2SQL": [{"A": 0.3318}, {"B": 0.3922}, {"C": 0.3806}, {"D": 0.098}], "OPENWORLD_QA": [{"A": 0.2674}, {"B": 0.2927}, {"C": 0.2065}, {"D": 0.1538}], "SOFTWARE": [{"A": 0.2594}, {"B": 0.4133}, {"C": 0.3288}, {"D": 0.3889}]}}
+{"model": "Memorybank (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.4359}, {"B": 0.4028}, {"C": 0.5}, {"D": 0.4167}], "EMBODIED_AI": [{"A": 0.1017}, {"B": 0.3908}, {"C": 0.131}, {"D": 0.0175}], "WEB": [{"A": 0.3393}, {"B": 0.5357}, {"C": 0.3571}, {"D": 0.375}], "TEXT2SQL": [{"A": 0.3139}, {"B": 0.3007}, {"C": 0.2612}, {"D": 0.0784}], "OPENWORLD_QA": [{"A": 0.2959}, {"B": 0.4421}, {"C": 0.2897}, {"D": 0.3667}], "SOFTWARE": [{"A": 0.4386}, {"B": 0.4545}, {"C": 0.2857}, {"D": 0.85}]}}
+{"model": "Simple mem (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.2083}, {"B": 0.2353}, {"C": 0.4615}, {"D": 0.25}], "EMBODIED_AI": [{"A": 0.0}, {"B": 0.1149}, {"C": 0.069}, {"D": 0.0}], "WEB": [{"A": 0.156}, {"B": 0.2099}, {"C": 0.0988}, {"D": 0.0943}], "TEXT2SQL": [{"A": 0.354}, {"B": 0.1644}, {"C": 0.1802}, {"D": 0.0717}], "OPENWORLD_QA": [{"A": 0.1224}, {"B": 0.1684}, {"C": 0.1121}, {"D": 0.0833}], "SOFTWARE": [{"A": 0.2538}, {"B": 0.2444}, {"C": 0.1333}, {"D": 0.3409}]}}
+{"model": "AMA-agent (Ours) (32B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.6471}, {"B": 0.7}, {"C": 0.6}, {"D": 0.6167}], "EMBODIED_AI": [{"A": 0.8033}, {"B": 0.5778}, {"C": 0.44}, {"D": 0.2169}], "WEB": [{"A": 0.53}, {"B": 0.68}, {"C": 0.4133}, {"D": 0.4}], "TEXT2SQL": [{"A": 0.7584}, {"B": 0.4615}, {"C": 0.6782}, {"D": 0.4118}], "OPENWORLD_QA": [{"A": 0.4482}, {"B": 0.5558}, {"C": 0.4612}, {"D": 0.39}], "SOFTWARE": [{"A": 0.6049}, {"B": 0.7268}, {"C": 0.4905}, {"D": 0.7778}]}}
+{"model": "BM25 (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.3966}, {"B": 0.4583}, {"C": 0.4615}, {"D": 0.3333}], "EMBODIED_AI": [{"A": 0.3443}, {"B": 0.3222}, {"C": 0.0733}, {"D": 0.0169}], "WEB": [{"A": 0.224}, {"B": 0.4409}, {"C": 0.2688}, {"D": 0.1803}], "TEXT2SQL": [{"A": 0.3498}, {"B": 0.451}, {"C": 0.3731}, {"D": 0.1765}], "OPENWORLD_QA": [{"A": 0.2347}, {"B": 0.2842}, {"C": 0.3364}, {"D": 0.2167}], "SOFTWARE": [{"A": 0.4292}, {"B": 0.6667}, {"C": 0.4247}, {"D": 0.75}]}}
+{"model": "Qwen3-Embedding-4B (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.4397}, {"B": 0.3056}, {"C": 0.6026}, {"D": 0.35}], "EMBODIED_AI": [{"A": 0.4262}, {"B": 0.3111}, {"C": 0.1133}, {"D": 0.0}], "WEB": [{"A": 0.392}, {"B": 0.4301}, {"C": 0.2258}, {"D": 0.2459}], "TEXT2SQL": [{"A": 0.5291}, {"B": 0.451}, {"C": 0.3582}, {"D": 0.2059}], "OPENWORLD_QA": [{"A": 0.4184}, {"B": 0.3895}, {"C": 0.3925}, {"D": 0.25}], "SOFTWARE": [{"A": 0.5283}, {"B": 0.68}, {"C": 0.3699}, {"D": 0.5972}]}}
+{"model": "GRAPHRAG (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.4818}, {"B": 0.3182}, {"C": 0.4324}, {"D": 0.4464}], "EMBODIED_AI": [{"A": 0.0196}, {"B": 0.28}, {"C": 0.088}, {"D": 0.0}], "WEB": [{"A": 0.304}, {"B": 0.3656}, {"C": 0.3441}, {"D": 0.2787}], "TEXT2SQL": [{"A": 0.1502}, {"B": 0.1156}, {"C": 0.1769}, {"D": 0.0408}], "OPENWORLD_QA": [{"A": 0.2}, {"B": 0.2283}, {"C": 0.1748}, {"D": 0.2241}], "SOFTWARE": [{"A": 0.2784}, {"B": 0.2754}, {"C": 0.209}, {"D": 0.3333}]}}
+{"model": "Hipporag2 (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.4528}, {"B": 0.4343}, {"C": 0.4946}, {"D": 0.4429}], "EMBODIED_AI": [{"A": 0.1026}, {"B": 0.2807}, {"C": 0.1053}, {"D": 0.0}], "WEB": [{"A": 0.3191}, {"B": 0.3525}, {"C": 0.3908}, {"D": 0.2573}], "TEXT2SQL": [{"A": 0.4316}, {"B": 0.5217}, {"C": 0.534}, {"D": 0.2759}], "OPENWORLD_QA": [{"A": 0.4316}, {"B": 0.5217}, {"C": 0.534}, {"D": 0.2759}], "SOFTWARE": [{"A": 0.3497}, {"B": 0.5538}, {"C": 0.3871}, {"D": 0.5}]}}
+{"model": "Memagent (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.1638}, {"B": 0.2917}, {"C": 0.3077}, {"D": 0.25}], "EMBODIED_AI": [{"A": 0.0164}, {"B": 0.1889}, {"C": 0.04}, {"D": 0.0169}], "WEB": [{"A": 0.2}, {"B": 0.2903}, {"C": 0.1828}, {"D": 0.1475}], "TEXT2SQL": [{"A": 0.2018}, {"B": 0.2484}, {"C": 0.1343}, {"D": 0.1078}], "OPENWORLD_QA": [{"A": 0.1735}, {"B": 0.1895}, {"C": 0.0935}, {"D": 0.0167}], "SOFTWARE": [{"A": 0.3396}, {"B": 0.5333}, {"C": 0.4795}, {"D": 0.3194}]}}
+{"model": "Mem1 (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.1624}, {"B": 0.2222}, {"C": 0.2692}, {"D": 0.15}], "EMBODIED_AI": [{"A": 0.0328}, {"B": 0.2444}, {"C": 0.0}, {"D": 0.0169}], "WEB": [{"A": 0.24}, {"B": 0.2151}, {"C": 0.1828}, {"D": 0.0656}], "TEXT2SQL": [{"A": 0.1076}, {"B": 0.1242}, {"C": 0.1045}, {"D": 0.0392}], "OPENWORLD_QA": [{"A": 0.1224}, {"B": 0.1895}, {"C": 0.0841}, {"D": 0.15}], "SOFTWARE": [{"A": 0.1934}, {"B": 0.16}, {"C": 0.1233}, {"D": 0.2917}]}}
+{"model": "Amem (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.431}, {"B": 0.5278}, {"C": 0.5641}, {"D": 0.4333}], "EMBODIED_AI": [{"A": 0.082}, {"B": 0.3667}, {"C": 0.1667}, {"D": 0.0847}], "WEB": [{"A": 0.336}, {"B": 0.4946}, {"C": 0.3333}, {"D": 0.3115}], "TEXT2SQL": [{"A": 0.3991}, {"B": 0.4575}, {"C": 0.3134}, {"D": 0.1667}], "OPENWORLD_QA": [{"A": 0.1327}, {"B": 0.3158}, {"C": 0.2056}, {"D": 0.2833}], "SOFTWARE": [{"A": 0.4198}, {"B": 0.5333}, {"C": 0.3973}, {"D": 0.5833}]}}
+{"model": "Mem0 (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.5741}, {"B": 0.5152}, {"C": 0.5946}, {"D": 0.5}], "EMBODIED_AI": [{"A": 0.0164}, {"B": 0.3889}, {"C": 0.02}, {"D": 0.0169}], "WEB": [{"A": 0.32}, {"B": 0.4086}, {"C": 0.2688}, {"D": 0.2623}], "TEXT2SQL": [{"A": 0.1928}, {"B": 0.2418}, {"C": 0.1493}, {"D": 0.1765}], "OPENWORLD_QA": [{"A": 0.1837}, {"B": 0.1895}, {"C": 0.1589}, {"D": 0.2167}], "SOFTWARE": [{"A": 0.3984}, {"B": 0.3191}, {"C": 0.2667}, {"D": 0.3864}]}}
+{"model": "Memgpt (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.4538}, {"B": 0.4889}, {"C": 0.5444}, {"D": 0.4833}], "EMBODIED_AI": [{"A": 0.1803}, {"B": 0.1778}, {"C": 0.0333}, {"D": 0.0}], "WEB": [{"A": 0.36}, {"B": 0.4516}, {"C": 0.2903}, {"D": 0.2459}], "TEXT2SQL": [{"A": 0.2063}, {"B": 0.3007}, {"C": 0.1866}, {"D": 0.0196}], "OPENWORLD_QA": [{"A": 0.0918}, {"B": 0.1158}, {"C": 0.0841}, {"D": 0.1}], "SOFTWARE": [{"A": 0.4151}, {"B": 0.5333}, {"C": 0.3562}, {"D": 0.625}]}}
+{"model": "Mem-alpha (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.292}, {"B": 0.3768}, {"C": 0.4211}, {"D": 0.2759}], "EMBODIED_AI": [{"A": 0.0984}, {"B": 0.2}, {"C": 0.0333}, {"D": 0.0}], "WEB": [{"A": 0.1736}, {"B": 0.2889}, {"C": 0.2}, {"D": 0.2881}], "TEXT2SQL": [{"A": 0.2242}, {"B": 0.3922}, {"C": 0.2537}, {"D": 0.0294}], "OPENWORLD_QA": [{"A": 0.2093}, {"B": 0.2439}, {"C": 0.2174}, {"D": 0.1346}], "SOFTWARE": [{"A": 0.217}, {"B": 0.32}, {"C": 0.1781}, {"D": 0.5139}]}}
+{"model": "Memorag (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.5}, {"B": 0.5278}, {"C": 0.641}, {"D": 0.55}], "EMBODIED_AI": [{"A": 0.1639}, {"B": 0.1333}, {"C": 0.02}, {"D": 0.0169}], "WEB": [{"A": 0.232}, {"B": 0.3763}, {"C": 0.1935}, {"D": 0.2295}], "TEXT2SQL": [{"A": 0.6637}, {"B": 0.6928}, {"C": 0.5896}, {"D": 0.3039}], "OPENWORLD_QA": [{"A": 0.2449}, {"B": 0.3789}, {"C": 0.3271}, {"D": 0.2333}], "SOFTWARE": [{"A": 0.4481}, {"B": 0.68}, {"C": 0.4384}, {"D": 0.6111}]}}
+{"model": "Memorybank (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.2821}, {"B": 0.3056}, {"C": 0.3974}, {"D": 0.3167}], "EMBODIED_AI": [{"A": 0.0656}, {"B": 0.2889}, {"C": 0.06}, {"D": 0.0169}], "WEB": [{"A": 0.28}, {"B": 0.3871}, {"C": 0.2473}, {"D": 0.4426}], "TEXT2SQL": [{"A": 0.2735}, {"B": 0.2157}, {"C": 0.209}, {"D": 0.1078}], "OPENWORLD_QA": [{"A": 0.2857}, {"B": 0.3895}, {"C": 0.243}, {"D": 0.4167}], "SOFTWARE": [{"A": 0.25}, {"B": 0.3467}, {"C": 0.2192}, {"D": 0.625}]}}
+{"model": "Simple mem (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.3613}, {"B": 0.1556}, {"C": 0.3556}, {"D": 0.3333}], "EMBODIED_AI": [{"A": 0.0328}, {"B": 0.2778}, {"C": 0.08}, {"D": 0.0}], "WEB": [{"A": 0.1538}, {"B": 0.3333}, {"C": 0.1379}, {"D": 0.193}], "TEXT2SQL": [{"A": 0.1302}, {"B": 0.069}, {"C": 0.0846}, {"D": 0.0102}], "OPENWORLD_QA": [{"A": 0.1939}, {"B": 0.3053}, {"C": 0.215}, {"D": 0.3}], "SOFTWARE": [{"A": 0.2028}, {"B": 0.2133}, {"C": 0.1507}, {"D": 0.2361}]}}
+{"model": "AMA-agent (Ours) (8B)", "Date": "2026-03-03", "verified": true, "Score": {"Game": [{"A": 0.5798}, {"B": 0.5}, {"C": 0.4556}, {"D": 0.6167}], "EMBODIED_AI": [{"A": 0.7705}, {"B": 0.4}, {"C": 0.1667}, {"D": 0.0}], "WEB": [{"A": 0.52}, {"B": 0.5269}, {"C": 0.4086}, {"D": 0.3934}], "TEXT2SQL": [{"A": 0.713}, {"B": 0.3529}, {"C": 0.597}, {"D": 0.3137}], "OPENWORLD_QA": [{"A": 0.475}, {"B": 0.4875}, {"C": 0.4556}, {"D": 0.32}], "SOFTWARE": [{"A": 0.5534}, {"B": 0.6164}, {"C": 0.5352}, {"D": 0.6286}]}}

submission.py

@@ -1,14 +1,28 @@
 """
 Submission handling module for AMA-Bench Leaderboard
-Manages model/agent submission processing and scoring
+
+Submission format:
+{
+    "episode_id": str,
+    "question_uuid_list": list[str],          # required - UUIDs that map answers to groundtruth
+    "answer_list": list[str],                 # required - same length as question_uuid_list
+    "llm_as_judge_score_list": list[bool]     # optional - same length as answer_list
+}
+
+Scoring logic:
+- Uses llm_as_judge_score_list (true/false) from the submission
+- Maps each question to its domain and capability (A/B/C/D) via groundtruth metadata
+- Computes per-domain, per-capability accuracy
+- Writes entry to data/agent.jsonl or data/model.jsonl (verified=False by default)
 """
 
 import json
 import os
 import datetime
 from email.utils import parseaddr
+from collections import defaultdict
+from typing import Dict, List, Tuple, Optional
 
-# Optional imports with fallbacks
 try:
     from content import format_error, format_warning, format_log
 except ImportError:
@@ -17,85 +31,80 @@ except ImportError:
     def format_log(msg): return f"✅ {msg}"
 
 
-def validate_submission_format(data):
-    """
-    Validate submission data format.
-
-    Expected format:
-    {
-        "episode_id": str,
-        "answer_list": list,
-        "llm_as_judge_score_list": list  # Optional, list of true/false
-    }
+# ---------------------------------------------------------------------------
+# Validation
+# ---------------------------------------------------------------------------
 
-    Args:
-        data: Dictionary containing submission data
+def validate_submission_format(data: dict) -> Tuple[bool, str]:
+    """
+    Validate a single submission record.
 
-    Returns:
-        (is_valid, error_msg): Tuple of validation result
+    Required fields: episode_id, question_uuid_list, answer_list
+    Optional fields: llm_as_judge_score_list, reasoning_trace
     """
     if not isinstance(data, dict):
         return False, "Submission must be a JSON object"
 
-    # Check required fields
+    # episode_id
     if "episode_id" not in data:
         return False, "Missing required field: episode_id"
-
-    if "answer_list" not in data:
-        return False, "Missing required field: answer_list"
-
-    # Validate episode_id
     if not isinstance(data["episode_id"], str) or not data["episode_id"].strip():
         return False, "episode_id must be a non-empty string"
 
-    # Validate answer_list
-    if not isinstance(data["answer_list"], list):
-        return False, "answer_list must be a list"
-
-    if not data["answer_list"]:
-        return False, "answer_list cannot be empty"
-
-    # Validate llm_as_judge_score_list if present
-    if "llm_as_judge_score_list" in data:
-        score_list = data["llm_as_judge_score_list"]
-        if not isinstance(score_list, list):
-            return False, "llm_as_judge_score_list must be a list"
-
-        if len(score_list) != len(data["answer_list"]):
-            return False, "llm_as_judge_score_list length must match answer_list length"
-
-        # Check that all values are boolean
-        for score in score_list:
-            if not isinstance(score, bool):
-                return False, "All values in llm_as_judge_score_list must be true or false"
+    # answer_list
+    if "answer_list" not in data:
+        return False, "Missing required field: answer_list"
+    if not isinstance(data["answer_list"], list) or not data["answer_list"]:
+        return False, "answer_list must be a non-empty list"
+
+    # question_uuid_list
+    if "question_uuid_list" not in data:
+        return False, "Missing required field: question_uuid_list"
+    if not isinstance(data["question_uuid_list"], list):
+        return False, "question_uuid_list must be a list"
+    if len(data["question_uuid_list"]) != len(data["answer_list"]):
+        return False, (
+            f"question_uuid_list length ({len(data['question_uuid_list'])}) must match "
+            f"answer_list length ({len(data['answer_list'])})"
+        )
+    for i, q in enumerate(data["question_uuid_list"]):
+        if not isinstance(q, str) or not q.strip():
+            return False, f"question_uuid_list[{i}] must be a non-empty string"
+
+    # llm_as_judge_score_list (required)
+    if "llm_as_judge_score_list" not in data:
+        return False, "Missing required field: llm_as_judge_score_list"
+    score_list = data["llm_as_judge_score_list"]
+    if not isinstance(score_list, list):
+        return False, "llm_as_judge_score_list must be a list"
+    if len(score_list) != len(data["answer_list"]):
+        return False, (
+            f"llm_as_judge_score_list length ({len(score_list)}) must match "
+            f"answer_list length ({len(data['answer_list'])})"
+        )
+    for i, score in enumerate(score_list):
+        if not isinstance(score, bool):
+            return False, f"llm_as_judge_score_list[{i}] must be true or false (boolean)"
 
     return True, ""
 
 
-def validate_submission_file(file_path):
-    """
-    Validate and load submission file.
+def validate_submission_file(file_path: str) -> Tuple[bool, str, List[dict]]:
+    """Validate and load a JSONL submission file."""
+    if not os.path.exists(file_path):
+        return False, "File not found", []
+    if not file_path.endswith(".jsonl"):
+        return False, "File must be in JSONL format (.jsonl)", []
 
-    Args:
-        file_path: Path to JSONL submission file
+    submissions = []
+    seen_ids = set()
 
-    Returns:
-        (is_valid, error_msg, submissions): Tuple of validation result and submissions
-    """
     try:
-        if not os.path.exists(file_path):
-            return False, "File not found", []
-
-        if not file_path.endswith('.jsonl'):
-            return False, "File must be in JSONL format (.jsonl)", []
-
-        submissions = []
-        with open(file_path, 'r', encoding='utf-8') as f:
+        with open(file_path, "r", encoding="utf-8") as f:
             for line_num, line in enumerate(f, 1):
                 line = line.strip()
                 if not line:
                     continue
-
                 try:
                     data = json.loads(line)
                 except json.JSONDecodeError as e:
@@ -105,244 +114,409 @@ def validate_submission_file(file_path):
                 if not is_valid:
                     return False, f"Validation error on line {line_num}: {error_msg}", []
 
+                episode_id = data["episode_id"]
+                if episode_id in seen_ids:
+                    return False, f"Duplicate episode_id '{episode_id}' on line {line_num}", []
+                seen_ids.add(episode_id)
                 submissions.append(data)
 
         if not submissions:
             return False, "File is empty or contains no valid submissions", []
-
         return True, "", submissions
 
     except Exception as e:
         return False, f"Error reading file: {e}", []
 
 
-def calculate_accuracy(submission, groundtruth_map):
+# ---------------------------------------------------------------------------
+# Groundtruth loading
+# ---------------------------------------------------------------------------
+
+def load_groundtruth_metadata(dataset_name: str = "Pettingllms/AMA-bench",
+                               token: str = None) -> Dict[str, dict]:
     """
-    Calculate accuracy for a submission against groundtruth.
+    Load groundtruth metadata. Returns a dict with two sub-dicts:
 
-    Args:
-        submission: Submission data dictionary
-        groundtruth_map: Dictionary mapping episode_id to groundtruth answers
+    {
+        "episode_domain": {
+            "episode_id": "GAME" | "TEXT2SQL" | ...
+        },
+        "question_cap": {
+            "question_uuid": "A" | "B" | "C" | "D"
+        }
+    }
 
-    Returns:
-        Dictionary with accuracy metrics by domain and capability
+    - episode_domain: episode_id -> domain  (used even when question uuid doesn't match)
+    - question_cap:   question_uuid -> capability letter
     """
-    episode_id = submission["episode_id"]
-    answer_list = submission["answer_list"]
-
-    if episode_id not in groundtruth_map:
-        return None
+    episode_domain: Dict[str, str] = {}
+    question_cap:   Dict[str, str] = {}
+
+    def _index_rows(rows):
+        for row in rows:
+            episode_id = str(row.get("episode_id", ""))
+            domain = row.get("domain", "UNKNOWN").upper()
+            episode_domain[episode_id] = domain
+            for qa in row.get("qa_pairs", []):
+                question_uuid = qa.get("question_uuid", "").strip()
+                if not question_uuid:
+                    continue
+                cap_letter = _normalize_cap(qa.get("type", "A"))
+                question_cap[question_uuid] = cap_letter
 
-    groundtruth = groundtruth_map[episode_id]
-    gt_answers = groundtruth.get("answer_list", [])
+    # --- Try HuggingFace ---
+    try:
+        from datasets import load_dataset, VerificationMode
+        dataset = load_dataset(
+            dataset_name, split="test", token=token,
+            verification_mode=VerificationMode.NO_CHECKS,
+        )
+        _index_rows(dataset)
+        print(f"[groundtruth] Loaded {len(episode_domain)} episodes, "
+              f"{len(question_cap)} Q&A entries from HuggingFace (indexed by question_uuid).")
+        return {"episode_domain": episode_domain, "question_cap": question_cap}
+    except Exception as hf_err:
+        print(f"[groundtruth] HuggingFace failed ({hf_err}), trying local fallback…")
+
+    # --- Local fallback ---
+    for local_path in ["test/open_end_qa_set.jsonl", "data/open_end_qa_set.jsonl"]:
+        if not os.path.exists(local_path):
+            continue
+        try:
+            rows = []
+            with open(local_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if line:
+                        rows.append(json.loads(line))
+            _index_rows(rows)
+            print(f"[groundtruth] Loaded {len(episode_domain)} episodes, "
+                  f"{len(question_cap)} Q&A entries from {local_path} (indexed by question_uuid).")
+            return {"episode_domain": episode_domain, "question_cap": question_cap}
+        except Exception as e:
+            print(f"[groundtruth] Error reading {local_path}: {e}")
+
+    print("[groundtruth] WARNING: No groundtruth metadata available.")
+    return {"episode_domain": {}, "question_cap": {}}
+
+
+def _normalize_cap(cap: str) -> str:
+    """Normalize capability label to single letter A/B/C/D."""
+    mapping = {
+        "A": "A", "Recall": "A",
+        "B": "B", "Causal Inference": "B", "Causal": "B",
+        "C": "C", "State Updating": "C", "State": "C",
+        "D": "D", "State Abstraction": "D", "Abstraction": "D",
+    }
+    return mapping.get(cap.strip(), "A")
 
-    if len(answer_list) != len(gt_answers):
-        return None
 
-    # Calculate per-question correctness
-    correct_count = sum(1 for pred, gt in zip(answer_list, gt_answers) if pred == gt)
-    total_count = len(answer_list)
+# ---------------------------------------------------------------------------
+# Scoring
+# ---------------------------------------------------------------------------
 
-    # Get domain and capability info from groundtruth
-    domain = groundtruth.get("domain", "UNKNOWN")
-    capabilities = groundtruth.get("capabilities", [])  # List of A/B/C/D for each question
+VALID_DOMAINS = {"TEXT2SQL", "SOFTWARE", "WEB", "GAME", "EMBODIED_AI", "OPENWORLD_QA"}
+VALID_CAPS = ["A", "B", "C", "D"]
 
-    # Calculate per-capability accuracy
-    capability_scores = {"A": [], "B": [], "C": [], "D": []}
 
-    for i, (pred, gt, cap) in enumerate(zip(answer_list, gt_answers, capabilities)):
-        if cap in capability_scores:
-            capability_scores[cap].append(1 if pred == gt else 0)
+def compute_scores_from_submissions(
+    submissions: List[dict],
+    groundtruth_meta: Dict[str, dict],
+) -> Dict:
+    """
+    Compute per-domain, per-capability accuracy using llm_as_judge_score_list.
 
-    result = {
-        "episode_id": episode_id,
-        "domain": domain,
-        "total_correct": correct_count,
-        "total_questions": total_count,
-        "accuracy": correct_count / total_count if total_count > 0 else 0,
-        "capability_scores": {}
+    Each question is matched to groundtruth via question_uuid.
+    Score structure matches agent.jsonl / model.jsonl:
+    {
+        "TEXT2SQL": [{"A": 0.xx}, {"B": 0.xx}, {"C": 0.xx}, {"D": 0.xx}],
+        ...
     }
+    """
+    # domain -> capability -> [scores]
+    domain_cap_scores: Dict[str, Dict[str, List[float]]] = defaultdict(
+        lambda: defaultdict(list)
+    )
+
+    scored_questions = 0
+    skipped_episodes = 0     # no judge scores
+    unmatched_questions = 0  # question cap not found (domain still resolved via episode)
+
+    # Unpack the two indexes from groundtruth metadata
+    episode_domain: Dict[str, str] = groundtruth_meta.get("episode_domain", {})
+    question_cap:   Dict[str, str] = groundtruth_meta.get("question_cap", {})
+
+    for sub in submissions:
+        episode_id = str(sub["episode_id"])
+        question_uuid_list = sub["question_uuid_list"]
+        judge_scores = sub.get("llm_as_judge_score_list")
+
+        # Resolve domain via episode_id
+        domain = episode_domain.get(episode_id, "UNKNOWN").upper()
+
+        for i, question_uuid in enumerate(question_uuid_list):
+            if i >= len(judge_scores):
+                break
+
+            # Resolve capability via question_uuid
+            cap = question_cap.get(question_uuid.strip())
+            if cap is None:
+                unmatched_questions += 1
+                continue
+
+            score = 1.0 if judge_scores[i] is True else 0.0
+            domain_cap_scores[domain][cap].append(score)
+            scored_questions += 1
+
+    # Build Score dict — always include all 6 known domains
+    score_dict: Dict[str, List[dict]] = {}
+    for domain in sorted(VALID_DOMAINS | set(domain_cap_scores.keys())):
+        cap_data = domain_cap_scores.get(domain, {})
+        score_dict[domain] = [
+            {cap: round(sum(cap_data[cap]) / len(cap_data[cap]), 4)
+             if cap_data.get(cap) else 0.0}
+            for cap in VALID_CAPS
+        ]
+
+    # Coverage warning
+    coverage_warning = None
+    parts = []
+    if skipped_episodes:
+        parts.append(f"{skipped_episodes} episode(s) had no llm_as_judge_score_list")
+    if unmatched_questions:
+        parts.append(f"{unmatched_questions} question(s) not matched in groundtruth")
+    if parts:
+        coverage_warning = "; ".join(parts)
 
-    # Calculate accuracy per capability
-    for cap, scores in capability_scores.items():
-        if scores:
-            result["capability_scores"][cap] = {
-                "correct": sum(scores),
-                "total": len(scores),
-                "accuracy": sum(scores) / len(scores)
-            }
-
-    return result
-
+    return {
+        "Score": score_dict,
+        "scored_questions": scored_questions,
+        "skipped_episodes": skipped_episodes,
+        "unmatched_questions": unmatched_questions,
+        "coverage_warning": coverage_warning,
+    }
 
-def process_submission(file_path, model_name, submission_type, organisation, verified=False):
-    """
-    Process a submission file and save results.
 
-    Args:
-        file_path: Path to submission JSONL file
-        model_name: Name of model/agent
-        submission_type: "model" or "agent"
-        organisation: Organisation name
-        verified: Whether this is an official verified submission
+# ---------------------------------------------------------------------------
+# Leaderboard update
+# ---------------------------------------------------------------------------
 
-    Returns:
-        Dictionary with submission results
-    """
-    # Validate file
-    is_valid, error_msg, submissions = validate_submission_file(file_path)
-    if not is_valid:
-        return {"success": False, "error": error_msg}
-
-    # Create submission directory
-    submission_dir = f"submissions/{organisation}_{model_name}"
-    os.makedirs(submission_dir, exist_ok=True)
-
-    timestamp = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
-
-    # Save submission file
-    saved_file = f"{submission_dir}/submission_{timestamp}.jsonl"
-    with open(saved_file, 'w', encoding='utf-8') as f:
-        for sub in submissions:
-            f.write(json.dumps(sub, ensure_ascii=False) + "\n")
-
-    # Create metadata
-    metadata = {
-        "model": model_name if submission_type == "model" else None,
-        "agent_name": model_name if submission_type == "agent" else None,
-        "model_family": "",
-        "submission_type": submission_type,
-        "organisation": organisation,
-        "Date": datetime.datetime.today().strftime('%Y-%m-%d'),
-        "timestamp": timestamp,
-        "verified": verified,
-        "submission_count": len(submissions),
-        "file_path": saved_file
-    }
+def update_leaderboard_data(
+    model_or_agent_name: str,
+    model_family: str,
+    submission_type: str,
+    organisation: str,
+    score_dict: Dict,
+    verified: bool = False,
+) -> bool:
+    """Append a scored entry to data/agent.jsonl or data/model.jsonl."""
+    try:
+        os.makedirs("data", exist_ok=True)
+        data_file = "data/agent.jsonl" if submission_type == "agent" else "data/model.jsonl"
+        name_key = "agent_name" if submission_type == "agent" else "model"
 
-    metadata_file = f"{submission_dir}/metadata_{timestamp}.json"
-    with open(metadata_file, 'w', encoding='utf-8') as f:
-        json.dump(metadata, f, indent=2, ensure_ascii=False)
+        entry = {
+            name_key: model_or_agent_name,
+            "model_family": model_family,
+            "Date": datetime.datetime.today().strftime("%Y-%m-%d"),
+            "verified": verified,
+            "Score": score_dict,
+        }
+        with open(data_file, "a", encoding="utf-8") as f:
+            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
 
-    return {
-        "success": True,
-        "submission_dir": submission_dir,
-        "metadata_file": metadata_file,
-        "saved_file": saved_file,
-        "submission_count": len(submissions),
-        "metadata": metadata
-    }
+        print(f"[leaderboard] Appended to {data_file}: {model_or_agent_name}")
+        return True
+    except Exception as e:
+        print(f"[leaderboard] Error: {e}")
+        import traceback; traceback.print_exc()
+        return False
 
 
-def add_new_submission(model, submission_type, url, file, organisation, mail, model_family=""):
-    """
-    Process and evaluate a new model/agent submission.
-
-    Args:
-        model: Model/agent name
-        submission_type: "Model" or "Agent"
-        url: URL to model/agent information
-        file: Uploaded file object
-        organisation: Organisation name
-        mail: Contact email
-        model_family: Model family (optional)
-
-    Returns:
-        String with formatted result message
-    """
+# ---------------------------------------------------------------------------
+# Main entry point
+# ---------------------------------------------------------------------------
+
+def add_new_submission(
+    model: str,
+    submission_type: str,
+    url: str,
+    file,
+    organisation: str,
+    mail: str,
+    model_family: str = "",
+) -> str:
+    """Validate, score, and record a new submission."""
     try:
-        # Validate inputs
         if file is None:
             return format_warning("Please attach a submission file.")
-
         _, parsed_mail = parseaddr(mail)
         if "@" not in parsed_mail:
             return format_warning("Please provide a valid email address.")
-
         if not model or not submission_type or not organisation:
             return format_warning("Please fill in all required fields.")
 
-        print(f"Processing submission from {organisation}/{model}")
+        print(f"[submission] Processing {organisation}/{model} ({submission_type})")
 
-        # Validate file format
         is_valid, error_msg, submissions = validate_submission_file(file.name)
         if not is_valid:
             return format_error(error_msg)
 
-        print(f"✓ Validated {len(submissions)} episode submissions")
-
-        # Process submission (without scoring for now - scoring will be done weekly)
-        result = process_submission(
-            file.name,
-            model,
-            submission_type.lower(),
-            organisation,
-            verified=False  # User submissions start as unverified
-        )
-
-        if not result["success"]:
-            return format_error(result["error"])
-
-        # Format success message
-        message = f"✅ **Submission received successfully!**\n\n"
-        message += f"**{'Agent' if submission_type.lower() == 'agent' else 'Model'}:** {model}\n"
-        message += f"**Organisation:** {organisation}\n"
-        message += f"**Episodes Submitted:** {result['submission_count']}\n\n"
-        message += f"**Submission ID:** {result['metadata']['timestamp']}\n\n"
-        message += "📊 **Important Notes:**\n"
-        message += "- Submissions are evaluated **once per week** using our LLM-as-Judge system\n"
-        message += "- You can submit **once per week** per user\n"
-        message += "- Official scores (verified=true) are computed by our evaluation system\n"
-        message += "- You can also run your own evaluation if you have the groundtruth\n"
-        message += "- Results will appear on the leaderboard after official evaluation\n\n"
-        message += f"Your submission has been saved to: `{result['submission_dir']}`"
-
-        return format_log(message)
-
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        return format_error(f"An error occurred during submission: {str(e)}")
-
-
-def update_leaderboard_data(submission_metadata, scores):
-    """
-    Update leaderboard JSONL files with new submission results.
-
-    Args:
-        submission_metadata: Metadata from processed submission
-        scores: Computed scores by domain and capability
-
-    Returns:
-        Boolean indicating success
-    """
-    try:
-        submission_type = submission_metadata["submission_type"]
-
-        # Determine which file to update
-        if submission_type == "agent":
-            data_file = "data/agent.jsonl"
-        else:
-            data_file = "data/model.jsonl"
-
-        # Create new entry
-        entry = {
-            "model" if submission_type == "model" else "agent_name": submission_metadata.get("model") or submission_metadata.get("agent_name"),
-            "model_family": submission_metadata.get("model_family", ""),
-            "Date": submission_metadata["Date"],
-            "verified": submission_metadata["verified"],
-            "Score": scores
+        print(f"[submission] Validated {len(submissions)} episode submissions")
+
+        groundtruth_meta = load_groundtruth_metadata()
+        score_result = compute_scores_from_submissions(submissions, groundtruth_meta)
+        score_dict = score_result["Score"]
+
+        # Save raw submission
+        submission_dir = f"submissions/{organisation}_{model}"
+        os.makedirs(submission_dir, exist_ok=True)
+        timestamp = datetime.datetime.today().strftime("%Y%m%d_%H%M%S")
+        saved_file = f"{submission_dir}/submission_{timestamp}.jsonl"
+        with open(saved_file, "w", encoding="utf-8") as f_out:
+            for sub in submissions:
+                f_out.write(json.dumps(sub, ensure_ascii=False) + "\n")
+
+        # Save metadata
+        metadata = {
+            "model" if submission_type.lower() == "model" else "agent_name": model,
+            "model_family": model_family,
+            "submission_type": submission_type.lower(),
+            "organisation": organisation,
+            "url": url,
+            "mail": parsed_mail,
+            "Date": datetime.datetime.today().strftime("%Y-%m-%d"),
+            "timestamp": timestamp,
+            "verified": False,
+            "submission_count": len(submissions),
+            "scored_questions": score_result["scored_questions"],
+            "skipped_episodes": score_result["skipped_episodes"],
+            "unmatched_questions": score_result["unmatched_questions"],
+            "file_path": saved_file,
         }
+        with open(f"{submission_dir}/metadata_{timestamp}.json", "w", encoding="utf-8") as f_meta:
+            json.dump(metadata, f_meta, indent=2, ensure_ascii=False)
+
+        # Update leaderboard
+        updated = update_leaderboard_data(
+            model_or_agent_name=model,
+            model_family=model_family,
+            submission_type=submission_type.lower(),
+            organisation=organisation,
+            score_dict=score_dict,
+            verified=False,
+        )
+        if not updated:
+            return format_error("Submission validated but failed to update leaderboard data.")
+
+        type_label = "Agent" if submission_type.lower() == "agent" else "Model"
+
+        # Compute per-domain averages and overall avg
+        domain_order = ["TEXT2SQL", "SOFTWARE", "WEB", "GAME", "EMBODIED_AI", "OPENWORLD_QA"]
+        domain_avgs = {}
+        for dom in domain_order:
+            caps = score_dict.get(dom, [])
+            vals = [list(c.values())[0] for c in caps if c]
+            domain_avgs[dom] = sum(vals) / len(vals) if vals else 0.0
+        overall_avg = sum(domain_avgs.values()) / len(domain_avgs) if domain_avgs else 0.0
+
+        # Build domain rows — all colors explicit to override Gradio dark theme
+        TD = 'style="padding:8px 12px;text-align:center;color:#2c3e50;background:#ffffff;"'
+        TD_NAME = 'style="padding:8px 12px;font-weight:600;color:#1a1a2e;background:#ffffff;"'
+        TD_AVG = 'style="padding:8px 12px;text-align:center;font-weight:700;color:#0e9e7a;background:#ffffff;"'
+
+        dom_rows_html = ""
+        for i, dom in enumerate(domain_order):
+            caps = score_dict.get(dom, [])
+            cap_cells = "".join(
+                f'<td {TD}>{list(c.values())[0]*100:.1f}%</td>'
+                for c in caps
+            )
+            avg = domain_avgs.get(dom, 0.0)
+            row_bg = "#f9fbff" if i % 2 == 0 else "#ffffff"
+            dom_rows_html += (
+                f'<tr style="border-bottom:1px solid #e8eef3;">'
+                f'<td style="padding:8px 12px;font-weight:600;color:#1a1a2e;background:{row_bg};">{dom}</td>'
+                f'<td style="padding:8px 12px;text-align:center;font-weight:700;color:#0e9e7a;background:{row_bg};">{avg*100:.2f}%</td>'
+                + "".join(
+                    f'<td style="padding:8px 12px;text-align:center;color:#2c3e50;background:{row_bg};">{list(c.values())[0]*100:.1f}%</td>'
+                    for c in caps
+                )
+                + '</tr>'
+            )
+
+        warning_html = (
+            '<div style="margin-top:12px;padding:10px 14px;background:#fff8e1;'
+            'border-left:4px solid #f0ad4e;border-radius:6px;font-size:13px;color:#7d5a00;">'
+            f'&#x26A0;&#xFE0F; {score_result["coverage_warning"]}</div>'
+            if score_result["coverage_warning"] else ""
+        )
 
-        # Append to file
-        with open(data_file, 'a', encoding='utf-8') as f:
-            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
-
-        print(f"✓ Updated {data_file}")
-        return True
+        result_html = (
+            '<div style="border:1px solid #c8e6c9;border-radius:12px;overflow:hidden;'
+            'margin-top:16px;font-family:-apple-system,BlinkMacSystemFont,sans-serif;'
+            'background:#ffffff;color:#1a1a2e;">'
+
+            # Header
+            '<div style="background:linear-gradient(135deg,#1abc9c,#16a085);padding:16px 22px;'
+            'display:flex;align-items:center;gap:12px;">'
+            '<span style="font-size:24px;">&#x2705;</span>'
+            '<span style="color:#ffffff;font-size:18px;font-weight:700;letter-spacing:0.3px;">'
+            'Submission Received Successfully</span>'
+            '</div>'
+
+            # Meta row
+            '<div style="padding:18px 22px;background:#f0faf7;display:flex;flex-wrap:wrap;'
+            'gap:28px;border-bottom:1px solid #d5eee8;">'
+            + "".join(
+                f'<div><div style="color:#6b8f85;font-size:11px;font-weight:600;'
+                f'letter-spacing:0.8px;text-transform:uppercase;">{label}</div>'
+                f'<div style="font-weight:700;font-size:15px;color:{color};margin-top:3px;">{value}</div></div>'
+                for label, value, color in [
+                    (type_label, model, "#1a1a2e"),
+                    ("Organisation", organisation, "#1a1a2e"),
+                    ("Episodes", str(len(submissions)), "#1a1a2e"),
+                    ("Questions Scored", str(score_result["scored_questions"]), "#1a1a2e"),
+                    ("Overall Avg", f"{overall_avg*100:.2f}%", "#0e9e7a"),
+                    ("Submission ID", timestamp, "#666"),
+                ]
+            )
+            + '</div>'
+
+            # Score table
+            '<div style="padding:18px 22px;background:#ffffff;">'
+            '<div style="font-size:13px;font-weight:600;color:#444;margin-bottom:12px;">'
+            '&#x1F4CA;&nbsp; Score Preview '
+            '<span style="font-weight:400;color:#888;">(self-reported · pending official verification)</span>'
+            '</div>'
+            '<div style="border-radius:8px;overflow:hidden;border:1px solid #e0eaf0;">'
+            '<table style="width:100%;border-collapse:collapse;font-size:13px;">'
+            '<thead>'
+            '<tr style="background:#e8f4f0;">'
+            '<th style="padding:9px 12px;text-align:left;color:#1a1a2e;font-weight:600;">Domain</th>'
+            '<th style="padding:9px 12px;text-align:center;color:#1a1a2e;font-weight:600;">Avg</th>'
+            '<th style="padding:9px 12px;text-align:center;color:#1a1a2e;font-weight:600;">Recall (A)</th>'
+            '<th style="padding:9px 12px;text-align:center;color:#1a1a2e;font-weight:600;">Causal Inf. (B)</th>'
+            '<th style="padding:9px 12px;text-align:center;color:#1a1a2e;font-weight:600;">State Upd. (C)</th>'
+            '<th style="padding:9px 12px;text-align:center;color:#1a1a2e;font-weight:600;">State Abs. (D)</th>'
+            '</tr>'
+            '</thead>'
+            f'<tbody>{dom_rows_html}</tbody>'
+            '</table>'
+            '</div>'
+            + warning_html +
+            '<div style="margin-top:14px;padding:10px 14px;background:#fffbea;border-radius:6px;'
+            'font-size:12px;color:#7d5a00;line-height:1.7;border-left:3px solid #f5c518;">'
+            '&#x2139;&#xFE0F;&nbsp; This is a <strong style="color:#5a3e00;">self-reported preview</strong> based on your '
+            '<code style="background:#f5e9b8;color:#5a3e00;padding:1px 4px;border-radius:3px;">llm_as_judge_score_list</code>. '
+            'Official scores will be recomputed by LLM-as-Judge — your entry will appear on the leaderboard after weekly verification.'
+            '</div>'
+            '</div>'
+            '</div>'
+            '</div>'
+        )
+        return result_html
 
     except Exception as e:
-        print(f"Error updating leaderboard data: {e}")
-        import traceback
-        traceback.print_exc()
-        return False
+        import traceback; traceback.print_exc()
+        return format_error(f"An error occurred: {str(e)}")