Add 3-seed formula-domain generalization study
Browse filesPublishes the behavior exact gate, context-contract audit, and three R-track formula adapters. These non-commercial research shadows fail the official writer-domain 92% gate and are explicitly excluded from product distillation.
- MANIFEST.json +277 -207
- MODEL_INDEX.json +31 -0
- README.md +22 -1
- models/research_formula_adapter/seed17/formula_adapter.pt +3 -0
- models/research_formula_adapter/seed31/formula_adapter.pt +3 -0
- models/research_formula_adapter/seed47/formula_adapter.pt +3 -0
- reports/RESEARCH_REPORT.md +27 -1
- reports/behavior_exact_gate_3seed.json +2102 -0
- reports/formula_adapter_h64_3seed_summary.json +92 -0
- reports/formula_adapter_h64_seed17.json +205 -0
- reports/formula_adapter_h64_seed31.json +205 -0
- reports/formula_adapter_h64_seed47.json +205 -0
- reports/formula_context_contract_3seed.json +106 -0
- scripts/audit_math_ink_06_behavior_exact_gate.py +255 -0
- scripts/audit_math_ink_06_formula_context_contract.py +308 -0
- scripts/summarize_math_ink_06_formula_adapter_seeds.py +96 -0
- scripts/train_math_ink_06_behavior_role.py +861 -0
- scripts/train_math_ink_06_formula_adapter.py +373 -0
MANIFEST.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
| 1 |
{
|
| 2 |
-
"schema": "aiflow-hf-research-snapshot-
|
| 3 |
-
"generated_at": "2026-07-
|
| 4 |
"track": "R_noncommercial_plus_rejected_P_proxy",
|
| 5 |
"product_validation": false,
|
| 6 |
"public_release": true,
|
| 7 |
"contains_raw_dataset": false,
|
| 8 |
-
"tests": "
|
| 9 |
"retracted_paths": [
|
| 10 |
"models/auxiliary/boundary_auxiliary_head.pt",
|
| 11 |
"models/auxiliary/seed17/boundary_joint_delta.pt",
|
|
@@ -14,344 +14,414 @@
|
|
| 14 |
],
|
| 15 |
"files": [
|
| 16 |
{
|
| 17 |
-
"path": ".
|
| 18 |
-
"bytes":
|
| 19 |
-
"sha256": "
|
| 20 |
},
|
| 21 |
{
|
| 22 |
-
"path": "
|
| 23 |
-
"bytes":
|
| 24 |
-
"sha256": "
|
| 25 |
},
|
| 26 |
{
|
| 27 |
-
"path": "
|
| 28 |
-
"bytes":
|
| 29 |
-
"sha256": "
|
| 30 |
},
|
| 31 |
{
|
| 32 |
-
"path": "
|
| 33 |
-
"bytes":
|
| 34 |
-
"sha256": "
|
| 35 |
},
|
| 36 |
{
|
| 37 |
-
"path": "
|
| 38 |
-
"bytes":
|
| 39 |
-
"sha256": "
|
| 40 |
},
|
| 41 |
{
|
| 42 |
-
"path": "
|
| 43 |
-
"bytes":
|
| 44 |
-
"sha256": "
|
| 45 |
},
|
| 46 |
{
|
| 47 |
-
"path": "
|
| 48 |
-
"bytes":
|
| 49 |
-
"sha256": "
|
| 50 |
},
|
| 51 |
{
|
| 52 |
-
"path": "
|
| 53 |
-
"bytes":
|
| 54 |
-
"sha256": "
|
| 55 |
},
|
| 56 |
{
|
| 57 |
-
"path": "
|
| 58 |
-
"bytes":
|
| 59 |
-
"sha256": "
|
| 60 |
},
|
| 61 |
{
|
| 62 |
-
"path": "
|
| 63 |
-
"bytes":
|
| 64 |
-
"sha256": "
|
| 65 |
},
|
| 66 |
{
|
| 67 |
-
"path": "
|
| 68 |
-
"bytes":
|
| 69 |
-
"sha256": "
|
| 70 |
},
|
| 71 |
{
|
| 72 |
-
"path": "
|
| 73 |
-
"bytes":
|
| 74 |
-
"sha256": "
|
| 75 |
},
|
| 76 |
{
|
| 77 |
-
"path": "
|
| 78 |
-
"bytes":
|
| 79 |
-
"sha256": "
|
| 80 |
},
|
| 81 |
{
|
| 82 |
-
"path": "
|
| 83 |
-
"bytes":
|
| 84 |
-
"sha256": "
|
| 85 |
},
|
| 86 |
{
|
| 87 |
-
"path": "
|
| 88 |
-
"bytes":
|
| 89 |
-
"sha256": "
|
| 90 |
},
|
| 91 |
{
|
| 92 |
-
"path": "
|
| 93 |
-
"bytes":
|
| 94 |
-
"sha256": "
|
| 95 |
},
|
| 96 |
{
|
| 97 |
-
"path": "
|
| 98 |
-
"bytes":
|
| 99 |
-
"sha256": "
|
| 100 |
},
|
| 101 |
{
|
| 102 |
-
"path": "
|
| 103 |
-
"bytes":
|
| 104 |
-
"sha256": "
|
| 105 |
},
|
| 106 |
{
|
| 107 |
-
"path": "
|
| 108 |
-
"bytes":
|
| 109 |
-
"sha256": "
|
| 110 |
},
|
| 111 |
{
|
| 112 |
-
"path": "
|
| 113 |
-
"bytes":
|
| 114 |
-
"sha256": "
|
| 115 |
},
|
| 116 |
{
|
| 117 |
-
"path": "
|
| 118 |
-
"bytes":
|
| 119 |
-
"sha256": "
|
| 120 |
},
|
| 121 |
{
|
| 122 |
-
"path": "
|
| 123 |
-
"bytes":
|
| 124 |
-
"sha256": "
|
| 125 |
},
|
| 126 |
{
|
| 127 |
-
"path": "
|
| 128 |
-
"bytes":
|
| 129 |
-
"sha256": "
|
| 130 |
},
|
| 131 |
{
|
| 132 |
-
"path": "
|
| 133 |
-
"bytes":
|
| 134 |
-
"sha256": "
|
| 135 |
},
|
| 136 |
{
|
| 137 |
-
"path": "
|
| 138 |
-
"bytes":
|
| 139 |
-
"sha256": "
|
| 140 |
},
|
| 141 |
{
|
| 142 |
-
"path": "
|
| 143 |
-
"bytes":
|
| 144 |
-
"sha256": "
|
| 145 |
},
|
| 146 |
{
|
| 147 |
-
"path": "
|
| 148 |
-
"bytes":
|
| 149 |
-
"sha256": "
|
| 150 |
},
|
| 151 |
{
|
| 152 |
-
"path": "
|
| 153 |
-
"bytes":
|
| 154 |
-
"sha256": "
|
| 155 |
},
|
| 156 |
{
|
| 157 |
-
"path": "
|
| 158 |
-
"bytes":
|
| 159 |
-
"sha256": "
|
| 160 |
},
|
| 161 |
{
|
| 162 |
-
"path": "
|
| 163 |
-
"bytes":
|
| 164 |
-
"sha256": "
|
| 165 |
},
|
| 166 |
{
|
| 167 |
-
"path": "
|
| 168 |
-
"bytes":
|
| 169 |
-
"sha256": "
|
| 170 |
},
|
| 171 |
{
|
| 172 |
-
"path": "
|
| 173 |
-
"bytes":
|
| 174 |
-
"sha256": "
|
| 175 |
},
|
| 176 |
{
|
| 177 |
-
"path": "
|
| 178 |
-
"bytes":
|
| 179 |
-
"sha256": "
|
| 180 |
},
|
| 181 |
{
|
| 182 |
-
"path": "
|
| 183 |
-
"bytes":
|
| 184 |
-
"sha256": "
|
| 185 |
},
|
| 186 |
{
|
| 187 |
-
"path": "
|
| 188 |
-
"bytes":
|
| 189 |
-
"sha256": "
|
| 190 |
},
|
| 191 |
{
|
| 192 |
-
"path": "
|
| 193 |
-
"bytes":
|
| 194 |
-
"sha256": "
|
| 195 |
},
|
| 196 |
{
|
| 197 |
-
"path": "
|
| 198 |
-
"bytes":
|
| 199 |
-
"sha256": "
|
| 200 |
},
|
| 201 |
{
|
| 202 |
-
"path": "
|
| 203 |
-
"bytes":
|
| 204 |
-
"sha256": "
|
| 205 |
},
|
| 206 |
{
|
| 207 |
-
"path": "
|
| 208 |
-
"bytes":
|
| 209 |
-
"sha256": "
|
| 210 |
},
|
| 211 |
{
|
| 212 |
-
"path": "
|
| 213 |
-
"bytes":
|
| 214 |
-
"sha256": "
|
| 215 |
},
|
| 216 |
{
|
| 217 |
-
"path": "
|
| 218 |
-
"bytes":
|
| 219 |
-
"sha256": "
|
| 220 |
},
|
| 221 |
{
|
| 222 |
-
"path": "
|
| 223 |
-
"bytes":
|
| 224 |
-
"sha256": "
|
| 225 |
},
|
| 226 |
{
|
| 227 |
-
"path": "
|
| 228 |
-
"bytes":
|
| 229 |
-
"sha256": "
|
| 230 |
},
|
| 231 |
{
|
| 232 |
-
"path": "
|
| 233 |
-
"bytes":
|
| 234 |
-
"sha256": "
|
| 235 |
},
|
| 236 |
{
|
| 237 |
-
"path": "
|
| 238 |
-
"bytes":
|
| 239 |
-
"sha256": "
|
| 240 |
},
|
| 241 |
{
|
| 242 |
-
"path": "
|
| 243 |
-
"bytes":
|
| 244 |
-
"sha256": "
|
| 245 |
},
|
| 246 |
{
|
| 247 |
-
"path": "
|
| 248 |
-
"bytes":
|
| 249 |
-
"sha256": "
|
| 250 |
},
|
| 251 |
{
|
| 252 |
-
"path": "
|
| 253 |
-
"bytes":
|
| 254 |
-
"sha256": "
|
| 255 |
},
|
| 256 |
{
|
| 257 |
-
"path": "
|
| 258 |
-
"bytes":
|
| 259 |
-
"sha256": "
|
| 260 |
},
|
| 261 |
{
|
| 262 |
-
"path": "
|
| 263 |
-
"bytes":
|
| 264 |
-
"sha256": "
|
| 265 |
},
|
| 266 |
{
|
| 267 |
-
"path": "
|
| 268 |
-
"bytes":
|
| 269 |
-
"sha256": "
|
| 270 |
},
|
| 271 |
{
|
| 272 |
-
"path": "
|
| 273 |
-
"bytes":
|
| 274 |
-
"sha256": "
|
| 275 |
},
|
| 276 |
{
|
| 277 |
-
"path": "
|
| 278 |
-
"bytes":
|
| 279 |
-
"sha256": "
|
| 280 |
},
|
| 281 |
{
|
| 282 |
-
"path": "
|
| 283 |
-
"bytes":
|
| 284 |
-
"sha256": "
|
| 285 |
},
|
| 286 |
{
|
| 287 |
-
"path": "
|
| 288 |
-
"bytes":
|
| 289 |
-
"sha256": "
|
| 290 |
},
|
| 291 |
{
|
| 292 |
-
"path": "
|
| 293 |
-
"bytes":
|
| 294 |
-
"sha256": "
|
| 295 |
},
|
| 296 |
{
|
| 297 |
-
"path": "
|
| 298 |
-
"bytes":
|
| 299 |
-
"sha256": "
|
| 300 |
},
|
| 301 |
{
|
| 302 |
-
"path": "
|
| 303 |
-
"bytes":
|
| 304 |
-
"sha256": "
|
| 305 |
},
|
| 306 |
{
|
| 307 |
-
"path": "
|
| 308 |
-
"bytes":
|
| 309 |
-
"sha256": "
|
| 310 |
},
|
| 311 |
{
|
| 312 |
-
"path": "
|
| 313 |
-
"bytes":
|
| 314 |
-
"sha256": "
|
| 315 |
},
|
| 316 |
{
|
| 317 |
-
"path": "
|
| 318 |
-
"bytes":
|
| 319 |
-
"sha256": "
|
| 320 |
},
|
| 321 |
{
|
| 322 |
-
"path": "
|
| 323 |
-
"bytes":
|
| 324 |
-
"sha256": "
|
| 325 |
},
|
| 326 |
{
|
| 327 |
-
"path": "
|
| 328 |
-
"bytes":
|
| 329 |
-
"sha256": "
|
| 330 |
},
|
| 331 |
{
|
| 332 |
-
"path": "
|
| 333 |
-
"bytes":
|
| 334 |
-
"sha256": "
|
| 335 |
},
|
| 336 |
{
|
| 337 |
-
"path": "
|
| 338 |
-
"bytes":
|
| 339 |
-
"sha256": "
|
| 340 |
},
|
| 341 |
{
|
| 342 |
-
"path": "
|
| 343 |
-
"bytes":
|
| 344 |
-
"sha256": "
|
| 345 |
},
|
| 346 |
{
|
| 347 |
-
"path": "
|
| 348 |
-
"bytes":
|
| 349 |
-
"sha256": "
|
| 350 |
},
|
| 351 |
{
|
| 352 |
-
"path": "
|
| 353 |
-
"bytes":
|
| 354 |
-
"sha256": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 355 |
}
|
| 356 |
]
|
| 357 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"schema": "aiflow-hf-research-snapshot-v13",
|
| 3 |
+
"generated_at": "2026-07-23T21:16:57.4916112Z",
|
| 4 |
"track": "R_noncommercial_plus_rejected_P_proxy",
|
| 5 |
"product_validation": false,
|
| 6 |
"public_release": true,
|
| 7 |
"contains_raw_dataset": false,
|
| 8 |
+
"tests": "298 passed",
|
| 9 |
"retracted_paths": [
|
| 10 |
"models/auxiliary/boundary_auxiliary_head.pt",
|
| 11 |
"models/auxiliary/seed17/boundary_joint_delta.pt",
|
|
|
|
| 14 |
],
|
| 15 |
"files": [
|
| 16 |
{
|
| 17 |
+
"path": "reports/p_boundary_joint_sharedfix_3seed_summary.json",
|
| 18 |
+
"bytes": 3056,
|
| 19 |
+
"sha256": "1ad9192d45218d4644f69a8f076487bd5fb36d5c2348f5225ee4ae06e9f4b127"
|
| 20 |
},
|
| 21 |
{
|
| 22 |
+
"path": "reports/RESEARCH_REPORT.md",
|
| 23 |
+
"bytes": 41248,
|
| 24 |
+
"sha256": "a2ff25796dab063f99b3790a64eac46e2c19f72a7e3f3a319abf8bfe6bc4fb50"
|
| 25 |
},
|
| 26 |
{
|
| 27 |
+
"path": "reports/p_boundary_joint_seed47.json",
|
| 28 |
+
"bytes": 6809,
|
| 29 |
+
"sha256": "fd4ddcf71209cf59443e7a252a8337d7812ff66fae9482867f6eb0a21666e064"
|
| 30 |
},
|
| 31 |
{
|
| 32 |
+
"path": "reports/p_boundary_joint_seed17.json",
|
| 33 |
+
"bytes": 6920,
|
| 34 |
+
"sha256": "7af3bb995041e5ea621d3c9535f3552e264860f12a7e6145637849f5cd0220a8"
|
| 35 |
},
|
| 36 |
{
|
| 37 |
+
"path": "reports/p_boundary_joint_seed31.json",
|
| 38 |
+
"bytes": 6803,
|
| 39 |
+
"sha256": "4f842852093ca46b1942c942fd5e475ca6c19f044ba9e27b6d372f1aed56056e"
|
| 40 |
},
|
| 41 |
{
|
| 42 |
+
"path": "scripts/audit_math_ink_06_online_error_consensus.py",
|
| 43 |
+
"bytes": 12174,
|
| 44 |
+
"sha256": "c562a8d0300aa358f9c3faaed274992c3548224347e1e875401eec93953d0807"
|
| 45 |
},
|
| 46 |
{
|
| 47 |
+
"path": "scripts/benchmark_math_ink_06_composite.py",
|
| 48 |
+
"bytes": 7618,
|
| 49 |
+
"sha256": "6c5347fe68114116e0dfd29e55e3682f47b3ce42af1412009ab7883e2076887c"
|
| 50 |
},
|
| 51 |
{
|
| 52 |
+
"path": "scripts/audit_math_ink_06_local_baseline_overmerge.py",
|
| 53 |
+
"bytes": 11900,
|
| 54 |
+
"sha256": "f32a18c8e7d3a0ea36379293e6be88bfb509b7df99846c38149fb8458e6cc994"
|
| 55 |
},
|
| 56 |
{
|
| 57 |
+
"path": "scripts/audit_math_ink_06_behavior_exact_gate.py",
|
| 58 |
+
"bytes": 10759,
|
| 59 |
+
"sha256": "79feee00110eebb9429c1577df43e8747831cec897b0955e64dc5b5c5da1a018"
|
| 60 |
},
|
| 61 |
{
|
| 62 |
+
"path": "scripts/audit_math_ink_06_formula_context_contract.py",
|
| 63 |
+
"bytes": 12602,
|
| 64 |
+
"sha256": "1a81c46465958e2ae5f8f8ef1c09d7cf664e6ecb951d7706db5ccce4084d4220"
|
| 65 |
},
|
| 66 |
{
|
| 67 |
+
"path": "reports/online_casecontext_seed17.json",
|
| 68 |
+
"bytes": 33805,
|
| 69 |
+
"sha256": "050c00c14df4712f55b0c76118dd11615b9129b078d56bd987e329c9edee1bda"
|
| 70 |
},
|
| 71 |
{
|
| 72 |
+
"path": "reports/online_casecontext_seed31.json",
|
| 73 |
+
"bytes": 33576,
|
| 74 |
+
"sha256": "30e6ca83aa4c04d1d257b0221fea7ac726f8e09e91906fdecbc12a2bd4d4d34b"
|
| 75 |
},
|
| 76 |
{
|
| 77 |
+
"path": "reports/multistroke_family_guard_report.json",
|
| 78 |
+
"bytes": 15740,
|
| 79 |
+
"sha256": "27b31da71b9f39f7ded38aa1ce3694911af140d22854543d4ae2b5f9e709b479"
|
| 80 |
},
|
| 81 |
{
|
| 82 |
+
"path": "reports/local_baseline_guard_report.json",
|
| 83 |
+
"bytes": 24052,
|
| 84 |
+
"sha256": "8f209b2890245f01f437c26e7556e19fee76c057d449c9382e2d64a70a3764d6"
|
| 85 |
},
|
| 86 |
{
|
| 87 |
+
"path": "reports/local_baseline_overmerge_audit.json",
|
| 88 |
+
"bytes": 1237409,
|
| 89 |
+
"sha256": "33a9ccafebd559ca8b173f011e9a0a6f495d37b8fdb2693673663479d3457071"
|
| 90 |
},
|
| 91 |
{
|
| 92 |
+
"path": "reports/p_boundary_auxiliary_smoke.json",
|
| 93 |
+
"bytes": 16685,
|
| 94 |
+
"sha256": "9d250e5c75d4f104971551c07a702810bf94fb5b6e5e587035a624933766356e"
|
| 95 |
},
|
| 96 |
{
|
| 97 |
+
"path": "reports/p_boundary_device_stress_sharedfix_3seed.json",
|
| 98 |
+
"bytes": 10867,
|
| 99 |
+
"sha256": "3cad0c0e6aed77ebf2bd19a915fc3d898beb87d7b8a5e5ab29f2013b4184826e"
|
| 100 |
},
|
| 101 |
{
|
| 102 |
+
"path": "reports/online_family_fusion_calibration_3seed.json",
|
| 103 |
+
"bytes": 2861,
|
| 104 |
+
"sha256": "50b06e65e9220639c68a2686c4b123697e3f4f8411755b9d9e9222e7f39f3042"
|
| 105 |
},
|
| 106 |
{
|
| 107 |
+
"path": "reports/online_casecontext_seed47.json",
|
| 108 |
+
"bytes": 33727,
|
| 109 |
+
"sha256": "c3dfd328d9ff0a55eed2cb74657ad87e5e164968fddd7fae8c8fd92e137a4557"
|
| 110 |
},
|
| 111 |
{
|
| 112 |
+
"path": "reports/online_error_consensus_paired_test_3seed.json",
|
| 113 |
+
"bytes": 12267,
|
| 114 |
+
"sha256": "ec624e9c96d7fa767b438ca2d555b5b0930edc0b80716ee740229ae894711f52"
|
| 115 |
},
|
| 116 |
{
|
| 117 |
+
"path": "scripts/build_math_ink_06_litert_colab_bundle.py",
|
| 118 |
+
"bytes": 6862,
|
| 119 |
+
"sha256": "f008635ab59344005f65f1394f56275e4c92610cd25781a2d95243d060010718"
|
| 120 |
},
|
| 121 |
{
|
| 122 |
+
"path": "scripts/train_math_ink_06_p_boundary_joint.py",
|
| 123 |
+
"bytes": 18055,
|
| 124 |
+
"sha256": "6fa378ccb7d08f605d4976b6ae08247d0b1fa3f2cb7ce00ba5d35b79c6dfbb47"
|
| 125 |
},
|
| 126 |
{
|
| 127 |
+
"path": "src/ink06_export.py",
|
| 128 |
+
"bytes": 5348,
|
| 129 |
+
"sha256": "b9a3e73bcaa19e0a868b3daad533cc3a3b784a4b99d447c2d51f9b74644554a1"
|
| 130 |
},
|
| 131 |
{
|
| 132 |
+
"path": "scripts/train_math_ink_06_p_boundary_auxiliary.py",
|
| 133 |
+
"bytes": 20688,
|
| 134 |
+
"sha256": "70ca8a05dcbd48e79d812cd7ff44ee3d8479055f898143daa6d6ea7580b2973e"
|
| 135 |
},
|
| 136 |
{
|
| 137 |
+
"path": "scripts/train_math_ink_06_boundary_behavior_guard.py",
|
| 138 |
+
"bytes": 12461,
|
| 139 |
+
"sha256": "9fd410b402976416c12057a196babf6c98d971c41c23ae1a0e5f183c06a42b6e"
|
| 140 |
},
|
| 141 |
{
|
| 142 |
+
"path": "scripts/train_math_ink_06_formula_adapter.py",
|
| 143 |
+
"bytes": 14987,
|
| 144 |
+
"sha256": "8aafc8a5c5cc76cb4816e83e03fafe823f5c4ea59e5d5cdaceda7e19aceffc81"
|
| 145 |
},
|
| 146 |
{
|
| 147 |
+
"path": "src/structure_relations.py",
|
| 148 |
+
"bytes": 9671,
|
| 149 |
+
"sha256": "59edbdcbfebaab029612beed132f38b69c84487f5ba930fd169f3278e10323bb"
|
| 150 |
},
|
| 151 |
{
|
| 152 |
+
"path": "src/tray_joint.py",
|
| 153 |
+
"bytes": 19404,
|
| 154 |
+
"sha256": "160c2dd8db3dd4da77bdd21209a898b73fb0cb5a1b24e59d2c8bc54e159ea3e0"
|
| 155 |
},
|
| 156 |
{
|
| 157 |
+
"path": "src/segmentation_lattice.py",
|
| 158 |
+
"bytes": 13772,
|
| 159 |
+
"sha256": "8cc2dc1e13b56c21d53a76b9b863cdf5c8f8dd315da6eb18c1bd2b5b87c1a616"
|
| 160 |
},
|
| 161 |
{
|
| 162 |
+
"path": "src/math_ink_06.py",
|
| 163 |
+
"bytes": 59569,
|
| 164 |
+
"sha256": "1972ad82e7694e5c3c64776ef90568473bfdaf40b7ab5c46c973e482b98e48ad"
|
| 165 |
},
|
| 166 |
{
|
| 167 |
+
"path": "src/p_formula_gate06.py",
|
| 168 |
+
"bytes": 7910,
|
| 169 |
+
"sha256": "00c8cb3fd5b53f2d118886d7333fe75d62f2d7b7e7b161b662adbd43d1b14a7f"
|
| 170 |
},
|
| 171 |
{
|
| 172 |
+
"path": "scripts/export_math_ink_06_litert.py",
|
| 173 |
+
"bytes": 11229,
|
| 174 |
+
"sha256": "6dd7e606c0c8d2f150144ac3e3aa3de527d268be9391fc7fa2a43a3861468a6a"
|
| 175 |
},
|
| 176 |
{
|
| 177 |
+
"path": "scripts/preflight_math_ink_06_p_formula.py",
|
| 178 |
+
"bytes": 1664,
|
| 179 |
+
"sha256": "36aeeab2f2474ce874ef932b24eb062dfc3b9e6cf3c0d59410a052a9c9835a07"
|
| 180 |
},
|
| 181 |
{
|
| 182 |
+
"path": "scripts/evaluate_math_ink_06_p_boundary_device_stress.py",
|
| 183 |
+
"bytes": 10444,
|
| 184 |
+
"sha256": "bdd75047c4887a6e53720f1c84d224545844cdbd9b0239826a605d578b3c92a6"
|
| 185 |
},
|
| 186 |
{
|
| 187 |
+
"path": "scripts/calibrate_math_ink_06_online_family_fusion.py",
|
| 188 |
+
"bytes": 8546,
|
| 189 |
+
"sha256": "535c2b6197148fada9a1762f2c6db9a494e89b703a9e60b913348b7c7ffdc056"
|
| 190 |
},
|
| 191 |
{
|
| 192 |
+
"path": "scripts/evaluate_crohme_tray_joint_selector.py",
|
| 193 |
+
"bytes": 11671,
|
| 194 |
+
"sha256": "58c175a95cdeb9fc764fb96594a76888122b8f118f2f26da1283e743b1a7cd04"
|
| 195 |
},
|
| 196 |
{
|
| 197 |
+
"path": "scripts/sweep_math_ink_06_local_baseline_guard.py",
|
| 198 |
+
"bytes": 9063,
|
| 199 |
+
"sha256": "28cfc7b9b695256cfa9199d912d1d938d402292b88e9411d42fddb3508446d3b"
|
| 200 |
},
|
| 201 |
{
|
| 202 |
+
"path": "scripts/train_math_ink_06_behavior_role.py",
|
| 203 |
+
"bytes": 37596,
|
| 204 |
+
"sha256": "a06de3eb73de2b0fd6de73265cbe8987c525f76daa5ad1be86941ec9eb0fa505"
|
| 205 |
},
|
| 206 |
{
|
| 207 |
+
"path": "scripts/sweep_math_ink_06_component_competition_guard.py",
|
| 208 |
+
"bytes": 9084,
|
| 209 |
+
"sha256": "e2dee616e2c6ebdc32281624889017685648b7076515242e04a1452d98d1c7da"
|
| 210 |
},
|
| 211 |
{
|
| 212 |
+
"path": "scripts/summarize_math_ink_06_formula_adapter_seeds.py",
|
| 213 |
+
"bytes": 3851,
|
| 214 |
+
"sha256": "18a1de7d6aff20850472719c8a1afcf80fcd872b1109bfe97e462125bab243cf"
|
| 215 |
},
|
| 216 |
{
|
| 217 |
+
"path": "scripts/summarize_math_ink_06_p_boundary_joint.py",
|
| 218 |
+
"bytes": 4580,
|
| 219 |
+
"sha256": "c6ce78f521232bdb7e967b0c358232a72dd3f79747f9e32eb240d9bb10c6a422"
|
| 220 |
},
|
| 221 |
{
|
| 222 |
+
"path": "exports/seed17/export_manifest.json",
|
| 223 |
+
"bytes": 1069,
|
| 224 |
+
"sha256": "28ff6f17d2dfa19798d2e7a2de07e459a8abb2734fbd772f14ef9a65939ed63d"
|
| 225 |
},
|
| 226 |
{
|
| 227 |
+
"path": "exports/seed17/online.pt2",
|
| 228 |
+
"bytes": 9925815,
|
| 229 |
+
"sha256": "218d1f02864772bed3ac467590b2bcad957bb8fa61b8593089841b800ff9fe70"
|
| 230 |
},
|
| 231 |
{
|
| 232 |
+
"path": "contracts/aiflow_p_formula_v1.schema.json",
|
| 233 |
+
"bytes": 1903,
|
| 234 |
+
"sha256": "c78c8a612d2f24a5c09fe1aa158b365481102869c3f84b3a5510a895b416c424"
|
| 235 |
},
|
| 236 |
{
|
| 237 |
+
"path": "colab/aiflow_math_ink_06_litert_bundle.zip",
|
| 238 |
+
"bytes": 17277316,
|
| 239 |
+
"sha256": "5bb4d44d88ebd3a325e35ddfefd1b751c1a09151510e54ad189b5fe18e963701"
|
| 240 |
},
|
| 241 |
{
|
| 242 |
+
"path": "colab/aiflow_math_ink_06_litert_bundle.zip.manifest.json",
|
| 243 |
+
"bytes": 2978,
|
| 244 |
+
"sha256": "06251288705c15bea4e38285f42b86e9754aa3e7190a030278b36ba69a2d0a78"
|
| 245 |
},
|
| 246 |
{
|
| 247 |
+
"path": "models/research_formula_adapter/seed47/formula_adapter.pt",
|
| 248 |
+
"bytes": 92162,
|
| 249 |
+
"sha256": "19b9db82a8b4d3d1cfea78f382afb29acbde3da1669a7e1cabaf525471e2aae6"
|
| 250 |
},
|
| 251 |
{
|
| 252 |
+
"path": "models/seed17/base_378.pt",
|
| 253 |
+
"bytes": 8638290,
|
| 254 |
+
"sha256": "f3b0e53b50e41cf57d4881c501371b35648cf049d603984a0edf6148c2b62bd7"
|
| 255 |
},
|
| 256 |
{
|
| 257 |
+
"path": "models/research_formula_adapter/seed31/formula_adapter.pt",
|
| 258 |
+
"bytes": 92162,
|
| 259 |
+
"sha256": "e41abde69c05cbfa0569ebfea535ccb059c86c2574ac7a9c32caa5b73b9049f7"
|
| 260 |
},
|
| 261 |
{
|
| 262 |
+
"path": "exports/seed17/raster.pt2",
|
| 263 |
+
"bytes": 14374301,
|
| 264 |
+
"sha256": "79943f8860a34423d823306943441afb93a077f011ddf1a553b86b0b448b0934"
|
| 265 |
},
|
| 266 |
{
|
| 267 |
+
"path": "models/research_formula_adapter/seed17/formula_adapter.pt",
|
| 268 |
+
"bytes": 92162,
|
| 269 |
+
"sha256": "b5e685691aacf6940b1bb85e2a19c20bdfa0822d82e42d39a133ccb32b53645e"
|
| 270 |
},
|
| 271 |
{
|
| 272 |
+
"path": "pyproject.snapshot.toml",
|
| 273 |
+
"bytes": 669,
|
| 274 |
+
"sha256": "e9571da506b96d9ab6baa8f8e2c1d87cd8ee9fd0c880f3cf6646a99086ae3f09"
|
| 275 |
},
|
| 276 |
{
|
| 277 |
+
"path": "README.md",
|
| 278 |
+
"bytes": 15280,
|
| 279 |
+
"sha256": "b4e609dd231b2161dc2b80475cd01101e043ffba91774025b2162249fc8986b5"
|
| 280 |
},
|
| 281 |
{
|
| 282 |
+
"path": "NOTICE.md",
|
| 283 |
+
"bytes": 908,
|
| 284 |
+
"sha256": "a1a0bf4ebe1d5423cc461a9db656212bdcc56bf57a52f2f93c71afe6be67e239"
|
| 285 |
},
|
| 286 |
{
|
| 287 |
+
"path": ".hfignore",
|
| 288 |
+
"bytes": 293,
|
| 289 |
+
"sha256": "70aa1f68c49ffc3149181137ada5cf43c419ca0fdb0f9c3094f25dc011cbc4e2"
|
| 290 |
},
|
| 291 |
{
|
| 292 |
+
"path": "MODEL_INDEX.json",
|
| 293 |
+
"bytes": 4767,
|
| 294 |
+
"sha256": "2c652c35329c1c44a7f8132554731b29fd4fff4e8cbe45b3e5de862460f8c611"
|
| 295 |
},
|
| 296 |
{
|
| 297 |
+
"path": "colab/AIFlow_Math_Ink_06_LiteRT.ipynb",
|
| 298 |
+
"bytes": 3505,
|
| 299 |
+
"sha256": "7c0a2118bc84576957048e882df33681eb9a4e2fe832011b4d73117c0b57b0c1"
|
| 300 |
},
|
| 301 |
{
|
| 302 |
+
"path": "colab/aiflow_math_ink_06_litert_bundle.manifest.json",
|
| 303 |
+
"bytes": 2978,
|
| 304 |
+
"sha256": "d6876734bbb23394b40c3b484e2c44d9e06cf3f67de77d3f50205e554b41f758"
|
| 305 |
},
|
| 306 |
{
|
| 307 |
+
"path": "assets/stroke_encoding.svg",
|
| 308 |
+
"bytes": 3882,
|
| 309 |
+
"sha256": "66bf7659c2e86fd53e6583e7c5deb8d371432469c07d4e4ca0dd165b4a81f4ed"
|
| 310 |
},
|
| 311 |
{
|
| 312 |
+
"path": "artifacts/boundary_behavior_guard.joblib",
|
| 313 |
+
"bytes": 824191,
|
| 314 |
+
"sha256": "18aff472c1bdb25a519b1553364229c9869aed1f36a9d663cf88a1bc808fab8f"
|
| 315 |
},
|
| 316 |
{
|
| 317 |
+
"path": "assets/architecture.svg",
|
| 318 |
+
"bytes": 5695,
|
| 319 |
+
"sha256": "282865a2dae8325d86bd59a56a625224a6e1c176eaca5a286f25bcb20642df9b"
|
| 320 |
},
|
| 321 |
{
|
| 322 |
+
"path": "models/seed17/behavior_role_head.pt",
|
| 323 |
+
"bytes": 75794,
|
| 324 |
+
"sha256": "57ced82a2600bd342bba68ba0636ecc3fc98c3dc461e26a132b5264d0a3d95bd"
|
| 325 |
},
|
| 326 |
{
|
| 327 |
+
"path": "reports/component_competition_report.json",
|
| 328 |
+
"bytes": 22284,
|
| 329 |
+
"sha256": "4d3db5f71e5008381dd072f2c10aafb22e08f96e23f2a32f82a3a46abd9c70b9"
|
| 330 |
},
|
| 331 |
{
|
| 332 |
+
"path": "reports/composite_cpu_benchmark.json",
|
| 333 |
+
"bytes": 2949,
|
| 334 |
+
"sha256": "5f724d7eef306b54f7b1d110f1d5c738d3d9985cd9aa3bdb1aff3478d2e1e647"
|
| 335 |
},
|
| 336 |
{
|
| 337 |
+
"path": "reports/boundary_behavior_guard_report.json",
|
| 338 |
+
"bytes": 72053,
|
| 339 |
+
"sha256": "f55a230a9f0892b9676c71db46e9e527862b3d8da776dc43db767ce7a0f6a5c7"
|
| 340 |
},
|
| 341 |
{
|
| 342 |
+
"path": "reports/behavior_role_seed31.json",
|
| 343 |
+
"bytes": 45412,
|
| 344 |
+
"sha256": "c017752db64378be6fe8db69d79749a01b7d6aee8364aafb4b04577410dfb210"
|
| 345 |
},
|
| 346 |
{
|
| 347 |
+
"path": "reports/behavior_role_seed47.json",
|
| 348 |
+
"bytes": 30091,
|
| 349 |
+
"sha256": "3d4dcbe2f2b0afb483f992dacd62aab1089bdc41d55cc259bcb7fc21b5294db0"
|
| 350 |
},
|
| 351 |
{
|
| 352 |
+
"path": "reports/formula_adapter_h64_seed47.json",
|
| 353 |
+
"bytes": 5571,
|
| 354 |
+
"sha256": "8b1ac0f47a68263fa8d6fcf556a15b3592f5236f95e91a4652989160c9b340e1"
|
| 355 |
+
},
|
| 356 |
+
{
|
| 357 |
+
"path": "reports/formula_context_contract_3seed.json",
|
| 358 |
+
"bytes": 2905,
|
| 359 |
+
"sha256": "cd0ccaea5e92c719d384b6358700f1eb039cb73d4892999047bd76df7889ce10"
|
| 360 |
+
},
|
| 361 |
+
{
|
| 362 |
+
"path": "reports/formula_adapter_h64_seed31.json",
|
| 363 |
+
"bytes": 5572,
|
| 364 |
+
"sha256": "2f81f012e4e91e26b152f6b95510eb3f83d834232167f1f79d781f94e2e796f1"
|
| 365 |
+
},
|
| 366 |
+
{
|
| 367 |
+
"path": "reports/formula_adapter_h64_3seed_summary.json",
|
| 368 |
+
"bytes": 2733,
|
| 369 |
+
"sha256": "843bf894d137ee6c5cd5452732b08000a9e8d15e5d229c9d492fe70dd0a866ae"
|
| 370 |
+
},
|
| 371 |
+
{
|
| 372 |
+
"path": "reports/formula_adapter_h64_seed17.json",
|
| 373 |
+
"bytes": 5569,
|
| 374 |
+
"sha256": "e935240df61b31ccd8fb07a1e54c7486eeff9347f42150d279ee2fc68ed1b86b"
|
| 375 |
+
},
|
| 376 |
+
{
|
| 377 |
+
"path": "models/seed31/online_adapter.pt",
|
| 378 |
+
"bytes": 2875654,
|
| 379 |
+
"sha256": "fedbbab64cea6755e76e0633367ee45fbcc1d5726bb9dfdb7c5fe4bbc968bdf2"
|
| 380 |
+
},
|
| 381 |
+
{
|
| 382 |
+
"path": "models/seed47/base_378.pt",
|
| 383 |
+
"bytes": 8638290,
|
| 384 |
+
"sha256": "4131b21f6b00471c1c5a82aee4a2da45e599e42dc742fc1dd0dd8afb970f8939"
|
| 385 |
+
},
|
| 386 |
+
{
|
| 387 |
+
"path": "models/seed31/behavior_role_head.pt",
|
| 388 |
+
"bytes": 75794,
|
| 389 |
+
"sha256": "b7c22cf49320b90a6fd103d39cc3ff08ea7734ab83275aad66e2b529b749d9ae"
|
| 390 |
+
},
|
| 391 |
+
{
|
| 392 |
+
"path": "models/seed17/online_adapter.pt",
|
| 393 |
+
"bytes": 2875654,
|
| 394 |
+
"sha256": "8ea224d7bc107a4c8b6cb91c9de6598000dae463d3e6d39480c081ea728cc87f"
|
| 395 |
+
},
|
| 396 |
+
{
|
| 397 |
+
"path": "models/seed31/base_378.pt",
|
| 398 |
+
"bytes": 8638290,
|
| 399 |
+
"sha256": "e3c57bc8cbbd4542cacf7ab9200a9031dcdc5ee63ffbd6ad6c9f7ef4f0a64f37"
|
| 400 |
+
},
|
| 401 |
+
{
|
| 402 |
+
"path": "reports/behavior_role_3seed_summary.json",
|
| 403 |
+
"bytes": 3567,
|
| 404 |
+
"sha256": "b16049218d2d006d952f59f55642fa9f3fbfbc3cc3a730fbd7b075f507c3b251"
|
| 405 |
+
},
|
| 406 |
+
{
|
| 407 |
+
"path": "reports/behavior_role_seed17.json",
|
| 408 |
+
"bytes": 29047,
|
| 409 |
+
"sha256": "e2072fec1b1065777f0e9218c48d27f096b22f5545e7cc8d9962939282719561"
|
| 410 |
+
},
|
| 411 |
+
{
|
| 412 |
+
"path": "reports/behavior_exact_gate_3seed.json",
|
| 413 |
+
"bytes": 69615,
|
| 414 |
+
"sha256": "7b0890e13164f170be02f519e3ffc56930486e2018cde050434286f769bfce83"
|
| 415 |
+
},
|
| 416 |
+
{
|
| 417 |
+
"path": "models/seed47/behavior_role_head.pt",
|
| 418 |
+
"bytes": 75794,
|
| 419 |
+
"sha256": "f28fad4e0289e3bc9400829f41aba9bbd75c43d445b7462a84aadbdbe635c10b"
|
| 420 |
+
},
|
| 421 |
+
{
|
| 422 |
+
"path": "models/seed47/online_adapter.pt",
|
| 423 |
+
"bytes": 2876038,
|
| 424 |
+
"sha256": "46394eeebad3d27d3178f78e693e160700ba3f454d79044714b835bc51c07056"
|
| 425 |
}
|
| 426 |
]
|
| 427 |
}
|
MODEL_INDEX.json
CHANGED
|
@@ -92,6 +92,37 @@
|
|
| 92 |
"paired_test_top1": 0.8381808400154114,
|
| 93 |
"paired_test_gain_pp": 0.10576248168945312
|
| 94 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
"release_state": {
|
| 96 |
"track": "R_noncommercial_only",
|
| 97 |
"product_validation": false,
|
|
|
|
| 92 |
"paired_test_top1": 0.8381808400154114,
|
| 93 |
"paired_test_gain_pp": 0.10576248168945312
|
| 94 |
},
|
| 95 |
+
"behavior_exact_gate": {
|
| 96 |
+
"report": "reports/behavior_exact_gate_3seed.json",
|
| 97 |
+
"scope": "truth_group_conditional_behavior_targets",
|
| 98 |
+
"official_test_teacher_exact_top1_mean": 0.336483,
|
| 99 |
+
"official_test_final_exact_top1_mean": 0.472966,
|
| 100 |
+
"official_test_gain_pp_mean": 13.6483,
|
| 101 |
+
"product_validation": false
|
| 102 |
+
},
|
| 103 |
+
"formula_context_contract": {
|
| 104 |
+
"report": "reports/formula_context_contract_3seed.json",
|
| 105 |
+
"selected_mode": "formula",
|
| 106 |
+
"official_test_all_supported_exact_top1": 0.6303,
|
| 107 |
+
"official_test_all_supported_visual_family_top1": 0.6818,
|
| 108 |
+
"product_validation": false
|
| 109 |
+
},
|
| 110 |
+
"research_formula_adapter": {
|
| 111 |
+
"status": "research_shadow_only",
|
| 112 |
+
"training_track": "R_noncommercial_only",
|
| 113 |
+
"architecture": "zero_init_skeleton_trajectory_adapter_h64",
|
| 114 |
+
"checkpoints": {
|
| 115 |
+
"17": "models/research_formula_adapter/seed17/formula_adapter.pt",
|
| 116 |
+
"31": "models/research_formula_adapter/seed31/formula_adapter.pt",
|
| 117 |
+
"47": "models/research_formula_adapter/seed47/formula_adapter.pt"
|
| 118 |
+
},
|
| 119 |
+
"summary": "reports/formula_adapter_h64_3seed_summary.json",
|
| 120 |
+
"validation_visual_family_top1_mean": 0.9330453563714902,
|
| 121 |
+
"official_test_visual_family_top1_mean": 0.8817204301075269,
|
| 122 |
+
"visual_family_generalization_gap_pp_mean": 5.132492626396335,
|
| 123 |
+
"product_validation": false,
|
| 124 |
+
"distillation_allowed": false
|
| 125 |
+
},
|
| 126 |
"release_state": {
|
| 127 |
"track": "R_noncommercial_only",
|
| 128 |
"product_validation": false,
|
README.md
CHANGED
|
@@ -176,6 +176,22 @@ Paired source는 HWRT 내부 writer hash split과 UJI Pen v1/v2 writer-disjoint
|
|
| 176 |
| multiplication recall | 88.89% |
|
| 177 |
| ECE | 4.88% |
|
| 178 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
### Grouping boundary head
|
| 180 |
|
| 181 |
| 지표 | 이전 | boundary 적용 |
|
|
@@ -249,8 +265,10 @@ SymbolResult:
|
|
| 249 |
- 378-label paired writer/device-disjoint top-1 목표 92%에 아직 미달한다.
|
| 250 |
- 전체 수식 LaTeX, Tray decoder, gridding은 0.7 범위다.
|
| 251 |
- uppercase 역할 recall과 `O/0`, styled-letter hard family가 남은 병목이다.
|
|
|
|
|
|
|
| 252 |
- raster virtual-stroke 경로는 digit/Greek slice에서는 개선됐지만 378-label release gate를 통과하지 못했다.
|
| 253 |
-
- boundary/behavior
|
| 254 |
- P boundary auxiliary/joint는 shared-state 정정 후 세 seed release gate를 실패해 checkpoint를 철회했다.
|
| 255 |
- Device stress는 채택되지 않은 shadow auxiliary의 software perturbation 결과이며 실제 stylus/device-disjoint 성능 근거가 아니다.
|
| 256 |
- Android LiteRT 변환, PyTorch/LiteRT logit parity, 저가·중급·고급 기기 benchmark가 남아 있다.
|
|
@@ -282,6 +300,9 @@ PyTorch checkpoint와 joblib/pickle은 신뢰할 수 없는 출처에서 로드
|
|
| 282 |
- Composite CPU benchmark: [`reports/composite_cpu_benchmark.json`](reports/composite_cpu_benchmark.json)
|
| 283 |
- Online 3-seed 오류 합의: [`reports/online_error_consensus_paired_test_3seed.json`](reports/online_error_consensus_paired_test_3seed.json)
|
| 284 |
- Online family-fusion calibration: [`reports/online_family_fusion_calibration_3seed.json`](reports/online_family_fusion_calibration_3seed.json)
|
|
|
|
|
|
|
|
|
|
| 285 |
- LiteRT Colab notebook: [`colab/AIFlow_Math_Ink_06_LiteRT.ipynb`](colab/AIFlow_Math_Ink_06_LiteRT.ipynb)
|
| 286 |
- 실제 P formula schema: [`contracts/aiflow_p_formula_v1.schema.json`](contracts/aiflow_p_formula_v1.schema.json)
|
| 287 |
- 파일 checksum: [`MANIFEST.json`](MANIFEST.json)
|
|
|
|
| 176 |
| multiplication recall | 88.89% |
|
| 177 |
| ECE | 4.88% |
|
| 178 |
|
| 179 |
+
행동 head를 실제 teacher 출력 뒤에 연결하고 validation에서만 confidence threshold를 선택한 결과, CROHME official test의 대상 기호 exact top-1은 33.65%에서 47.30%로 평균 **13.65%p** 상승했다. Rewrite precision은 81.35%였다. 다만 같은 구간의 teacher visual-family top-1이 53.02%에 불과해, 행동 문맥만으로 잘못된 형태군을 복구할 수 없었다.
|
| 180 |
+
|
| 181 |
+
### R-track 연속 수식 어댑터
|
| 182 |
+
|
| 183 |
+
제품 trajectory encoder와 online adapter는 고정하고, CROHME 연속식의 정답 symbol group 위에서 92KB zero-init formula adapter만 학습했다. 이 checkpoint는 구조 검증용 **R_noncommercial_only** 모델이며 제품 weight 또는 distillation 입력이 아니다.
|
| 184 |
+
|
| 185 |
+
| 3-seed 지표 | 평균 | 최저 |
|
| 186 |
+
|---|---:|---:|
|
| 187 |
+
| writer-validation exact top-1 | 85.19% | 82.91% |
|
| 188 |
+
| writer-validation visual-family top-1 | **93.30%** | **93.03%** |
|
| 189 |
+
| official test exact top-1 | 82.38% | 81.13% |
|
| 190 |
+
| official test visual-family top-1 | 88.17% | 88.03% |
|
| 191 |
+
| visual-family 일반화 gap | 5.13%p | — |
|
| 192 |
+
|
| 193 |
+
Validation에서는 세 seed 모두 형태군 92%를 넘겨 현재 TCN 구조가 연속 수식에도 적응할 수 있음을 확인했다. 반면 unseen official test에서는 모두 실패했다. 현 병목은 모델 용량보다 writer/source domain 일반화이며, 다음 gate는 상용 허용 P-track 연속식의 writer/device-disjoint 재학습이다.
|
| 194 |
+
|
| 195 |
### Grouping boundary head
|
| 196 |
|
| 197 |
| 지표 | 이전 | boundary 적용 |
|
|
|
|
| 265 |
- 378-label paired writer/device-disjoint top-1 목표 92%에 아직 미달한다.
|
| 266 |
- 전체 수식 LaTeX, Tray decoder, gridding은 0.7 범위다.
|
| 267 |
- uppercase 역할 recall과 `O/0`, styled-letter hard family가 남은 병목이다.
|
| 268 |
+
- 행동 exact gate는 유효하지만, 연속식 teacher 형태군이 틀리면 역할 head가 복구할 수 없다.
|
| 269 |
+
- R-track formula adapter는 validation 형태군 93.30%를 달성했으나 official test 88.17%에 그쳐 제품 검증을 통과하지 못했다.
|
| 270 |
- raster virtual-stroke 경로는 digit/Greek slice에서는 개선됐지만 378-label release gate를 통과하지 못했다.
|
| 271 |
+
- boundary/behavior/formula adapter는 CROHME R-track 학습물이므로 제품 weight로 distill할 수 없다.
|
| 272 |
- P boundary auxiliary/joint는 shared-state 정정 후 세 seed release gate를 실패해 checkpoint를 철회했다.
|
| 273 |
- Device stress는 채택되지 않은 shadow auxiliary의 software perturbation 결과이며 실제 stylus/device-disjoint 성능 근거가 아니다.
|
| 274 |
- Android LiteRT 변환, PyTorch/LiteRT logit parity, 저가·중급·고급 기기 benchmark가 남아 있다.
|
|
|
|
| 300 |
- Composite CPU benchmark: [`reports/composite_cpu_benchmark.json`](reports/composite_cpu_benchmark.json)
|
| 301 |
- Online 3-seed 오류 합의: [`reports/online_error_consensus_paired_test_3seed.json`](reports/online_error_consensus_paired_test_3seed.json)
|
| 302 |
- Online family-fusion calibration: [`reports/online_family_fusion_calibration_3seed.json`](reports/online_family_fusion_calibration_3seed.json)
|
| 303 |
+
- 행동 exact gate 3-seed: [`reports/behavior_exact_gate_3seed.json`](reports/behavior_exact_gate_3seed.json)
|
| 304 |
+
- 수식 context 계약 감사: [`reports/formula_context_contract_3seed.json`](reports/formula_context_contract_3seed.json)
|
| 305 |
+
- R-track 수식 어댑터 3-seed: [`reports/formula_adapter_h64_3seed_summary.json`](reports/formula_adapter_h64_3seed_summary.json)
|
| 306 |
- LiteRT Colab notebook: [`colab/AIFlow_Math_Ink_06_LiteRT.ipynb`](colab/AIFlow_Math_Ink_06_LiteRT.ipynb)
|
| 307 |
- 실제 P formula schema: [`contracts/aiflow_p_formula_v1.schema.json`](contracts/aiflow_p_formula_v1.schema.json)
|
| 308 |
- 파일 checksum: [`MANIFEST.json`](MANIFEST.json)
|
models/research_formula_adapter/seed17/formula_adapter.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b5e685691aacf6940b1bb85e2a19c20bdfa0822d82e42d39a133ccb32b53645e
|
| 3 |
+
size 92162
|
models/research_formula_adapter/seed31/formula_adapter.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e41abde69c05cbfa0569ebfea535ccb059c86c2574ac7a9c32caa5b73b9049f7
|
| 3 |
+
size 92162
|
models/research_formula_adapter/seed47/formula_adapter.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:19b9db82a8b4d3d1cfea78f382afb29acbde3da1669a7e1cabaf525471e2aae6
|
| 3 |
+
size 92162
|
reports/RESEARCH_REPORT.md
CHANGED
|
@@ -448,7 +448,26 @@ Origin·writer·device·source가 둘 이상의 split에 나타나면 제품 평
|
|
| 448 |
|
| 449 |
Validation에서만 exact/family fusion weight를 0~0.30 범위로 고정한 결과 0.15가 선택됐다. Paired-test에는 한 번만 적용했으며 exact ensemble 83.71→83.82%(+0.11%p), macro-F1 +0.09%p였다. 개선 폭은 작지만 adapter checkpoint에 기록된 0.15가 Python 공개 runtime과 export graph에서 무시되던 결함을 수정했다. 새 seed-17 strict export는 대표 76개 top-1 100% 일치·logit 오차 0.0, online+raster 합계 24,300,116 bytes로 25MiB 안이다.
|
| 450 |
|
| 451 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 452 |
|
| 453 |
## 산출물
|
| 454 |
|
|
@@ -477,6 +496,10 @@ Validation에서만 exact/family fusion weight를 0~0.30 범위로 고정한 결
|
|
| 477 |
- `scripts/benchmark_math_ink_06_composite.py`
|
| 478 |
- `scripts/audit_math_ink_06_online_error_consensus.py`
|
| 479 |
- `scripts/calibrate_math_ink_06_online_family_fusion.py`
|
|
|
|
|
|
|
|
|
|
|
|
|
| 480 |
- `scripts/preflight_math_ink_06_p_formula.py`
|
| 481 |
- `scripts/analyze_crohme_lattice_failures.py`
|
| 482 |
- `tests/test_behavior_context06.py`
|
|
@@ -490,6 +513,9 @@ Validation에서만 exact/family fusion weight를 0~0.30 범위로 고정한 결
|
|
| 490 |
- `research/runs/math_ink_06_online_error_consensus_paired_test_3seed_20260724/report.json`
|
| 491 |
- `research/runs/math_ink_06_online_family_fusion_calibration_3seed_20260724/report.json`
|
| 492 |
- `research/runs/math_ink_06_composite_export_familyfusion_seed17_20260724/export_manifest.json`
|
|
|
|
|
|
|
|
|
|
| 493 |
- `research/AIFlow-MATH-INK-0.6-BEHAVIOR-CONTEXT-REPORT-20260724.md`
|
| 494 |
- `research/runs/math_ink_06_behavior_role_3seed_20260724/run_summary.json`
|
| 495 |
- `research/runs/math_ink_06_behavior_grouping_audit_20260724/report.json`
|
|
|
|
| 448 |
|
| 449 |
Validation에서만 exact/family fusion weight를 0~0.30 범위로 고정한 결과 0.15가 선택됐다. Paired-test에는 한 번만 적용했으며 exact ensemble 83.71→83.82%(+0.11%p), macro-F1 +0.09%p였다. 개선 폭은 작지만 adapter checkpoint에 기록된 0.15가 Python 공개 runtime과 export graph에서 무시되던 결함을 수정했다. 새 seed-17 strict export는 대표 76개 top-1 100% 일치·logit 오차 0.0, online+raster 합계 24,300,116 bytes로 25MiB 안이다.
|
| 450 |
|
| 451 |
+
### 연속 수식 visual-family 입력 계약과 R-track 적응
|
| 452 |
+
|
| 453 |
+
고립기호 paired-test에서 92.99%였던 visual-family가 실제 연속 수식에서도 유지되는지 CROHME 정답 symbol group 조건부로 다시 측정했다. 기존 행동 head의 93.33%는 정답 group과 `c/C`, `x/X/\times`, `z/Z` 역할이 주어진 조건부 값이어서, teacher 형태군 인식과 confidence abstain을 포함한 최종 exact를 직접 계산했다.
|
| 454 |
+
|
| 455 |
+
Seed별 threshold를 trainData writer-validation에서만 고정한 행동 exact gate는 official test에서 teacher exact를 평균 33.65→47.30%(+13.65%p)로 개선했다. Rewrite precision은 81.35%였다. 그러나 teacher visual-family 자체가 평균 53.02%여서 행동 head가 보지 못하는 앞단 domain 병목이 더 컸다.
|
| 456 |
+
|
| 457 |
+
Baseline 채널 주입을 `formula`, `missing`, `local_full`, `height_only` 네 방식으로 validation 비교했다. 전체 지원기호는 `missing` visual-family 76.55%가 가장 높았지만 행동 대상은 기존 `formula`가 65.52%로 가장 높아 전역 정규화 하나로 해결되지 않았다. 네 branch 중 하나라도 정답인 비배포 validation oracle도 전체 88.06%, 행동 대상 75.16%로 92%에 못 미쳤다. 따라서 경량 selector만 추가하는 안은 기각했다.
|
| 458 |
+
|
| 459 |
+
구조적 회복 가능성을 확인하기 위해 제품 encoder와 online adapter를 동결하고, 그 뒤에 hidden 64 residual TCN formula adapter만 CROHME R-track truth group으로 학습했다. Recipe는 family CE + exact CE 0.10, baseline-context dropout 0.30이며 seed 17 probe validation에서 고정한 뒤 seed 17·31·47로 재현했다.
|
| 460 |
+
|
| 461 |
+
| 지표 | 3-seed 평균 | 최솟값 |
|
| 462 |
+
|---|---:|---:|
|
| 463 |
+
| writer-validation exact | 85.19% | 82.91% |
|
| 464 |
+
| writer-validation visual-family | **93.30%** | **93.03%** |
|
| 465 |
+
| official test exact | 82.38% | 81.13% |
|
| 466 |
+
| official test visual-family | 88.17% | 88.03% |
|
| 467 |
+
|
| 468 |
+
세 seed 모두 writer-validation visual-family 92%를 넘겼으므로 연속식 적응 구조 자체는 유효하다. 하지만 official test는 세 seed 모두 92%를 실패했고 visual-family 일반화 gap은 평균 5.13%p다. 즉 잔여 병목은 모델 용량이나 정규화 selector보다 unseen writer/domain coverage다. 이 adapter는 CROHME R-track weight이므로 제품 checkpoint·distillation·LiteRT bundle에는 포함하지 않는다. 다음 제품 gate는 같은 recipe를 실제 P 연속식 writer/device-disjoint 데이터에 적용하는 것이다.
|
| 469 |
+
|
| 470 |
+
전체 회귀는 298개가 통과했다.
|
| 471 |
|
| 472 |
## 산출물
|
| 473 |
|
|
|
|
| 496 |
- `scripts/benchmark_math_ink_06_composite.py`
|
| 497 |
- `scripts/audit_math_ink_06_online_error_consensus.py`
|
| 498 |
- `scripts/calibrate_math_ink_06_online_family_fusion.py`
|
| 499 |
+
- `scripts/audit_math_ink_06_behavior_exact_gate.py`
|
| 500 |
+
- `scripts/audit_math_ink_06_formula_context_contract.py`
|
| 501 |
+
- `scripts/train_math_ink_06_formula_adapter.py`
|
| 502 |
+
- `scripts/summarize_math_ink_06_formula_adapter_seeds.py`
|
| 503 |
- `scripts/preflight_math_ink_06_p_formula.py`
|
| 504 |
- `scripts/analyze_crohme_lattice_failures.py`
|
| 505 |
- `tests/test_behavior_context06.py`
|
|
|
|
| 513 |
- `research/runs/math_ink_06_online_error_consensus_paired_test_3seed_20260724/report.json`
|
| 514 |
- `research/runs/math_ink_06_online_family_fusion_calibration_3seed_20260724/report.json`
|
| 515 |
- `research/runs/math_ink_06_composite_export_familyfusion_seed17_20260724/export_manifest.json`
|
| 516 |
+
- `research/runs/math_ink_06_behavior_exact_gate_3seed_20260724/report.json`
|
| 517 |
+
- `research/runs/math_ink_06_formula_context_contract_3seed_20260724/report.json`
|
| 518 |
+
- `research/runs/math_ink_06_formula_adapter_h64_3seed_20260724/run_summary.json`
|
| 519 |
- `research/AIFlow-MATH-INK-0.6-BEHAVIOR-CONTEXT-REPORT-20260724.md`
|
| 520 |
- `research/runs/math_ink_06_behavior_role_3seed_20260724/run_summary.json`
|
| 521 |
- `research/runs/math_ink_06_behavior_grouping_audit_20260724/report.json`
|
reports/behavior_exact_gate_3seed.json
ADDED
|
@@ -0,0 +1,2102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "R-MATH-INK-06-BEHAVIOR-EXACT-GATE-001",
|
| 3 |
+
"generated_at": "2026-07-23T20:53:17.837992+00:00",
|
| 4 |
+
"split_contract": "CROHME trainData writer-validation threshold; testDataGT one-shot",
|
| 5 |
+
"scope": "truth symbol grouping conditional; c/C, x/X/times, z/Z only",
|
| 6 |
+
"seeds": [
|
| 7 |
+
{
|
| 8 |
+
"adapter": "research\\runs\\math_ink_06_online_casecontext_refined_seed17_20260723\\skeleton_adapter.pt",
|
| 9 |
+
"behavior_head": "research\\runs\\math_ink_06_behavior_role_seed17_20260724\\behavior_role_head.pt",
|
| 10 |
+
"selected_threshold": 0.54,
|
| 11 |
+
"validation_selected": {
|
| 12 |
+
"threshold": 0.54,
|
| 13 |
+
"samples": 467,
|
| 14 |
+
"teacher_exact": 0.3576017130620985,
|
| 15 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 16 |
+
"final_exact": 0.715203426124197,
|
| 17 |
+
"gain_pp": 35.76017130620985,
|
| 18 |
+
"rewrites": 340,
|
| 19 |
+
"abstained": 127,
|
| 20 |
+
"beneficial": 170,
|
| 21 |
+
"harmful": 3,
|
| 22 |
+
"rewrite_precision": 0.9826589595375722
|
| 23 |
+
},
|
| 24 |
+
"validation_sweep": [
|
| 25 |
+
{
|
| 26 |
+
"threshold": 0.0,
|
| 27 |
+
"samples": 467,
|
| 28 |
+
"teacher_exact": 0.3576017130620985,
|
| 29 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 30 |
+
"final_exact": 0.715203426124197,
|
| 31 |
+
"gain_pp": 35.76017130620985,
|
| 32 |
+
"rewrites": 341,
|
| 33 |
+
"abstained": 126,
|
| 34 |
+
"beneficial": 170,
|
| 35 |
+
"harmful": 3,
|
| 36 |
+
"rewrite_precision": 0.9826589595375722
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"threshold": 0.02,
|
| 40 |
+
"samples": 467,
|
| 41 |
+
"teacher_exact": 0.3576017130620985,
|
| 42 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 43 |
+
"final_exact": 0.715203426124197,
|
| 44 |
+
"gain_pp": 35.76017130620985,
|
| 45 |
+
"rewrites": 341,
|
| 46 |
+
"abstained": 126,
|
| 47 |
+
"beneficial": 170,
|
| 48 |
+
"harmful": 3,
|
| 49 |
+
"rewrite_precision": 0.9826589595375722
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"threshold": 0.04,
|
| 53 |
+
"samples": 467,
|
| 54 |
+
"teacher_exact": 0.3576017130620985,
|
| 55 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 56 |
+
"final_exact": 0.715203426124197,
|
| 57 |
+
"gain_pp": 35.76017130620985,
|
| 58 |
+
"rewrites": 341,
|
| 59 |
+
"abstained": 126,
|
| 60 |
+
"beneficial": 170,
|
| 61 |
+
"harmful": 3,
|
| 62 |
+
"rewrite_precision": 0.9826589595375722
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"threshold": 0.06,
|
| 66 |
+
"samples": 467,
|
| 67 |
+
"teacher_exact": 0.3576017130620985,
|
| 68 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 69 |
+
"final_exact": 0.715203426124197,
|
| 70 |
+
"gain_pp": 35.76017130620985,
|
| 71 |
+
"rewrites": 341,
|
| 72 |
+
"abstained": 126,
|
| 73 |
+
"beneficial": 170,
|
| 74 |
+
"harmful": 3,
|
| 75 |
+
"rewrite_precision": 0.9826589595375722
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"threshold": 0.08,
|
| 79 |
+
"samples": 467,
|
| 80 |
+
"teacher_exact": 0.3576017130620985,
|
| 81 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 82 |
+
"final_exact": 0.715203426124197,
|
| 83 |
+
"gain_pp": 35.76017130620985,
|
| 84 |
+
"rewrites": 341,
|
| 85 |
+
"abstained": 126,
|
| 86 |
+
"beneficial": 170,
|
| 87 |
+
"harmful": 3,
|
| 88 |
+
"rewrite_precision": 0.9826589595375722
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"threshold": 0.1,
|
| 92 |
+
"samples": 467,
|
| 93 |
+
"teacher_exact": 0.3576017130620985,
|
| 94 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 95 |
+
"final_exact": 0.715203426124197,
|
| 96 |
+
"gain_pp": 35.76017130620985,
|
| 97 |
+
"rewrites": 341,
|
| 98 |
+
"abstained": 126,
|
| 99 |
+
"beneficial": 170,
|
| 100 |
+
"harmful": 3,
|
| 101 |
+
"rewrite_precision": 0.9826589595375722
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"threshold": 0.12,
|
| 105 |
+
"samples": 467,
|
| 106 |
+
"teacher_exact": 0.3576017130620985,
|
| 107 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 108 |
+
"final_exact": 0.715203426124197,
|
| 109 |
+
"gain_pp": 35.76017130620985,
|
| 110 |
+
"rewrites": 341,
|
| 111 |
+
"abstained": 126,
|
| 112 |
+
"beneficial": 170,
|
| 113 |
+
"harmful": 3,
|
| 114 |
+
"rewrite_precision": 0.9826589595375722
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"threshold": 0.14,
|
| 118 |
+
"samples": 467,
|
| 119 |
+
"teacher_exact": 0.3576017130620985,
|
| 120 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 121 |
+
"final_exact": 0.715203426124197,
|
| 122 |
+
"gain_pp": 35.76017130620985,
|
| 123 |
+
"rewrites": 341,
|
| 124 |
+
"abstained": 126,
|
| 125 |
+
"beneficial": 170,
|
| 126 |
+
"harmful": 3,
|
| 127 |
+
"rewrite_precision": 0.9826589595375722
|
| 128 |
+
},
|
| 129 |
+
{
|
| 130 |
+
"threshold": 0.16,
|
| 131 |
+
"samples": 467,
|
| 132 |
+
"teacher_exact": 0.3576017130620985,
|
| 133 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 134 |
+
"final_exact": 0.715203426124197,
|
| 135 |
+
"gain_pp": 35.76017130620985,
|
| 136 |
+
"rewrites": 341,
|
| 137 |
+
"abstained": 126,
|
| 138 |
+
"beneficial": 170,
|
| 139 |
+
"harmful": 3,
|
| 140 |
+
"rewrite_precision": 0.9826589595375722
|
| 141 |
+
},
|
| 142 |
+
{
|
| 143 |
+
"threshold": 0.18,
|
| 144 |
+
"samples": 467,
|
| 145 |
+
"teacher_exact": 0.3576017130620985,
|
| 146 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 147 |
+
"final_exact": 0.715203426124197,
|
| 148 |
+
"gain_pp": 35.76017130620985,
|
| 149 |
+
"rewrites": 341,
|
| 150 |
+
"abstained": 126,
|
| 151 |
+
"beneficial": 170,
|
| 152 |
+
"harmful": 3,
|
| 153 |
+
"rewrite_precision": 0.9826589595375722
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
"threshold": 0.2,
|
| 157 |
+
"samples": 467,
|
| 158 |
+
"teacher_exact": 0.3576017130620985,
|
| 159 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 160 |
+
"final_exact": 0.715203426124197,
|
| 161 |
+
"gain_pp": 35.76017130620985,
|
| 162 |
+
"rewrites": 341,
|
| 163 |
+
"abstained": 126,
|
| 164 |
+
"beneficial": 170,
|
| 165 |
+
"harmful": 3,
|
| 166 |
+
"rewrite_precision": 0.9826589595375722
|
| 167 |
+
},
|
| 168 |
+
{
|
| 169 |
+
"threshold": 0.22,
|
| 170 |
+
"samples": 467,
|
| 171 |
+
"teacher_exact": 0.3576017130620985,
|
| 172 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 173 |
+
"final_exact": 0.715203426124197,
|
| 174 |
+
"gain_pp": 35.76017130620985,
|
| 175 |
+
"rewrites": 341,
|
| 176 |
+
"abstained": 126,
|
| 177 |
+
"beneficial": 170,
|
| 178 |
+
"harmful": 3,
|
| 179 |
+
"rewrite_precision": 0.9826589595375722
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"threshold": 0.24,
|
| 183 |
+
"samples": 467,
|
| 184 |
+
"teacher_exact": 0.3576017130620985,
|
| 185 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 186 |
+
"final_exact": 0.715203426124197,
|
| 187 |
+
"gain_pp": 35.76017130620985,
|
| 188 |
+
"rewrites": 341,
|
| 189 |
+
"abstained": 126,
|
| 190 |
+
"beneficial": 170,
|
| 191 |
+
"harmful": 3,
|
| 192 |
+
"rewrite_precision": 0.9826589595375722
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"threshold": 0.26,
|
| 196 |
+
"samples": 467,
|
| 197 |
+
"teacher_exact": 0.3576017130620985,
|
| 198 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 199 |
+
"final_exact": 0.715203426124197,
|
| 200 |
+
"gain_pp": 35.76017130620985,
|
| 201 |
+
"rewrites": 341,
|
| 202 |
+
"abstained": 126,
|
| 203 |
+
"beneficial": 170,
|
| 204 |
+
"harmful": 3,
|
| 205 |
+
"rewrite_precision": 0.9826589595375722
|
| 206 |
+
},
|
| 207 |
+
{
|
| 208 |
+
"threshold": 0.28,
|
| 209 |
+
"samples": 467,
|
| 210 |
+
"teacher_exact": 0.3576017130620985,
|
| 211 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 212 |
+
"final_exact": 0.715203426124197,
|
| 213 |
+
"gain_pp": 35.76017130620985,
|
| 214 |
+
"rewrites": 341,
|
| 215 |
+
"abstained": 126,
|
| 216 |
+
"beneficial": 170,
|
| 217 |
+
"harmful": 3,
|
| 218 |
+
"rewrite_precision": 0.9826589595375722
|
| 219 |
+
},
|
| 220 |
+
{
|
| 221 |
+
"threshold": 0.3,
|
| 222 |
+
"samples": 467,
|
| 223 |
+
"teacher_exact": 0.3576017130620985,
|
| 224 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 225 |
+
"final_exact": 0.715203426124197,
|
| 226 |
+
"gain_pp": 35.76017130620985,
|
| 227 |
+
"rewrites": 341,
|
| 228 |
+
"abstained": 126,
|
| 229 |
+
"beneficial": 170,
|
| 230 |
+
"harmful": 3,
|
| 231 |
+
"rewrite_precision": 0.9826589595375722
|
| 232 |
+
},
|
| 233 |
+
{
|
| 234 |
+
"threshold": 0.32,
|
| 235 |
+
"samples": 467,
|
| 236 |
+
"teacher_exact": 0.3576017130620985,
|
| 237 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 238 |
+
"final_exact": 0.715203426124197,
|
| 239 |
+
"gain_pp": 35.76017130620985,
|
| 240 |
+
"rewrites": 341,
|
| 241 |
+
"abstained": 126,
|
| 242 |
+
"beneficial": 170,
|
| 243 |
+
"harmful": 3,
|
| 244 |
+
"rewrite_precision": 0.9826589595375722
|
| 245 |
+
},
|
| 246 |
+
{
|
| 247 |
+
"threshold": 0.34,
|
| 248 |
+
"samples": 467,
|
| 249 |
+
"teacher_exact": 0.3576017130620985,
|
| 250 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 251 |
+
"final_exact": 0.715203426124197,
|
| 252 |
+
"gain_pp": 35.76017130620985,
|
| 253 |
+
"rewrites": 341,
|
| 254 |
+
"abstained": 126,
|
| 255 |
+
"beneficial": 170,
|
| 256 |
+
"harmful": 3,
|
| 257 |
+
"rewrite_precision": 0.9826589595375722
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"threshold": 0.36,
|
| 261 |
+
"samples": 467,
|
| 262 |
+
"teacher_exact": 0.3576017130620985,
|
| 263 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 264 |
+
"final_exact": 0.715203426124197,
|
| 265 |
+
"gain_pp": 35.76017130620985,
|
| 266 |
+
"rewrites": 341,
|
| 267 |
+
"abstained": 126,
|
| 268 |
+
"beneficial": 170,
|
| 269 |
+
"harmful": 3,
|
| 270 |
+
"rewrite_precision": 0.9826589595375722
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"threshold": 0.38,
|
| 274 |
+
"samples": 467,
|
| 275 |
+
"teacher_exact": 0.3576017130620985,
|
| 276 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 277 |
+
"final_exact": 0.715203426124197,
|
| 278 |
+
"gain_pp": 35.76017130620985,
|
| 279 |
+
"rewrites": 341,
|
| 280 |
+
"abstained": 126,
|
| 281 |
+
"beneficial": 170,
|
| 282 |
+
"harmful": 3,
|
| 283 |
+
"rewrite_precision": 0.9826589595375722
|
| 284 |
+
},
|
| 285 |
+
{
|
| 286 |
+
"threshold": 0.4,
|
| 287 |
+
"samples": 467,
|
| 288 |
+
"teacher_exact": 0.3576017130620985,
|
| 289 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 290 |
+
"final_exact": 0.715203426124197,
|
| 291 |
+
"gain_pp": 35.76017130620985,
|
| 292 |
+
"rewrites": 341,
|
| 293 |
+
"abstained": 126,
|
| 294 |
+
"beneficial": 170,
|
| 295 |
+
"harmful": 3,
|
| 296 |
+
"rewrite_precision": 0.9826589595375722
|
| 297 |
+
},
|
| 298 |
+
{
|
| 299 |
+
"threshold": 0.42,
|
| 300 |
+
"samples": 467,
|
| 301 |
+
"teacher_exact": 0.3576017130620985,
|
| 302 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 303 |
+
"final_exact": 0.715203426124197,
|
| 304 |
+
"gain_pp": 35.76017130620985,
|
| 305 |
+
"rewrites": 341,
|
| 306 |
+
"abstained": 126,
|
| 307 |
+
"beneficial": 170,
|
| 308 |
+
"harmful": 3,
|
| 309 |
+
"rewrite_precision": 0.9826589595375722
|
| 310 |
+
},
|
| 311 |
+
{
|
| 312 |
+
"threshold": 0.44,
|
| 313 |
+
"samples": 467,
|
| 314 |
+
"teacher_exact": 0.3576017130620985,
|
| 315 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 316 |
+
"final_exact": 0.715203426124197,
|
| 317 |
+
"gain_pp": 35.76017130620985,
|
| 318 |
+
"rewrites": 341,
|
| 319 |
+
"abstained": 126,
|
| 320 |
+
"beneficial": 170,
|
| 321 |
+
"harmful": 3,
|
| 322 |
+
"rewrite_precision": 0.9826589595375722
|
| 323 |
+
},
|
| 324 |
+
{
|
| 325 |
+
"threshold": 0.46,
|
| 326 |
+
"samples": 467,
|
| 327 |
+
"teacher_exact": 0.3576017130620985,
|
| 328 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 329 |
+
"final_exact": 0.715203426124197,
|
| 330 |
+
"gain_pp": 35.76017130620985,
|
| 331 |
+
"rewrites": 341,
|
| 332 |
+
"abstained": 126,
|
| 333 |
+
"beneficial": 170,
|
| 334 |
+
"harmful": 3,
|
| 335 |
+
"rewrite_precision": 0.9826589595375722
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"threshold": 0.48,
|
| 339 |
+
"samples": 467,
|
| 340 |
+
"teacher_exact": 0.3576017130620985,
|
| 341 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 342 |
+
"final_exact": 0.715203426124197,
|
| 343 |
+
"gain_pp": 35.76017130620985,
|
| 344 |
+
"rewrites": 341,
|
| 345 |
+
"abstained": 126,
|
| 346 |
+
"beneficial": 170,
|
| 347 |
+
"harmful": 3,
|
| 348 |
+
"rewrite_precision": 0.9826589595375722
|
| 349 |
+
},
|
| 350 |
+
{
|
| 351 |
+
"threshold": 0.5,
|
| 352 |
+
"samples": 467,
|
| 353 |
+
"teacher_exact": 0.3576017130620985,
|
| 354 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 355 |
+
"final_exact": 0.715203426124197,
|
| 356 |
+
"gain_pp": 35.76017130620985,
|
| 357 |
+
"rewrites": 340,
|
| 358 |
+
"abstained": 127,
|
| 359 |
+
"beneficial": 170,
|
| 360 |
+
"harmful": 3,
|
| 361 |
+
"rewrite_precision": 0.9826589595375722
|
| 362 |
+
},
|
| 363 |
+
{
|
| 364 |
+
"threshold": 0.52,
|
| 365 |
+
"samples": 467,
|
| 366 |
+
"teacher_exact": 0.3576017130620985,
|
| 367 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 368 |
+
"final_exact": 0.715203426124197,
|
| 369 |
+
"gain_pp": 35.76017130620985,
|
| 370 |
+
"rewrites": 340,
|
| 371 |
+
"abstained": 127,
|
| 372 |
+
"beneficial": 170,
|
| 373 |
+
"harmful": 3,
|
| 374 |
+
"rewrite_precision": 0.9826589595375722
|
| 375 |
+
},
|
| 376 |
+
{
|
| 377 |
+
"threshold": 0.54,
|
| 378 |
+
"samples": 467,
|
| 379 |
+
"teacher_exact": 0.3576017130620985,
|
| 380 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 381 |
+
"final_exact": 0.715203426124197,
|
| 382 |
+
"gain_pp": 35.76017130620985,
|
| 383 |
+
"rewrites": 340,
|
| 384 |
+
"abstained": 127,
|
| 385 |
+
"beneficial": 170,
|
| 386 |
+
"harmful": 3,
|
| 387 |
+
"rewrite_precision": 0.9826589595375722
|
| 388 |
+
},
|
| 389 |
+
{
|
| 390 |
+
"threshold": 0.56,
|
| 391 |
+
"samples": 467,
|
| 392 |
+
"teacher_exact": 0.3576017130620985,
|
| 393 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 394 |
+
"final_exact": 0.7130620985010707,
|
| 395 |
+
"gain_pp": 35.54603854389722,
|
| 396 |
+
"rewrites": 339,
|
| 397 |
+
"abstained": 128,
|
| 398 |
+
"beneficial": 169,
|
| 399 |
+
"harmful": 3,
|
| 400 |
+
"rewrite_precision": 0.9825581395348837
|
| 401 |
+
},
|
| 402 |
+
{
|
| 403 |
+
"threshold": 0.58,
|
| 404 |
+
"samples": 467,
|
| 405 |
+
"teacher_exact": 0.3576017130620985,
|
| 406 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 407 |
+
"final_exact": 0.7130620985010707,
|
| 408 |
+
"gain_pp": 35.54603854389722,
|
| 409 |
+
"rewrites": 339,
|
| 410 |
+
"abstained": 128,
|
| 411 |
+
"beneficial": 169,
|
| 412 |
+
"harmful": 3,
|
| 413 |
+
"rewrite_precision": 0.9825581395348837
|
| 414 |
+
},
|
| 415 |
+
{
|
| 416 |
+
"threshold": 0.6,
|
| 417 |
+
"samples": 467,
|
| 418 |
+
"teacher_exact": 0.3576017130620985,
|
| 419 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 420 |
+
"final_exact": 0.7130620985010707,
|
| 421 |
+
"gain_pp": 35.54603854389722,
|
| 422 |
+
"rewrites": 338,
|
| 423 |
+
"abstained": 129,
|
| 424 |
+
"beneficial": 169,
|
| 425 |
+
"harmful": 3,
|
| 426 |
+
"rewrite_precision": 0.9825581395348837
|
| 427 |
+
},
|
| 428 |
+
{
|
| 429 |
+
"threshold": 0.62,
|
| 430 |
+
"samples": 467,
|
| 431 |
+
"teacher_exact": 0.3576017130620985,
|
| 432 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 433 |
+
"final_exact": 0.7130620985010707,
|
| 434 |
+
"gain_pp": 35.54603854389722,
|
| 435 |
+
"rewrites": 338,
|
| 436 |
+
"abstained": 129,
|
| 437 |
+
"beneficial": 169,
|
| 438 |
+
"harmful": 3,
|
| 439 |
+
"rewrite_precision": 0.9825581395348837
|
| 440 |
+
},
|
| 441 |
+
{
|
| 442 |
+
"threshold": 0.64,
|
| 443 |
+
"samples": 467,
|
| 444 |
+
"teacher_exact": 0.3576017130620985,
|
| 445 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 446 |
+
"final_exact": 0.7130620985010707,
|
| 447 |
+
"gain_pp": 35.54603854389722,
|
| 448 |
+
"rewrites": 338,
|
| 449 |
+
"abstained": 129,
|
| 450 |
+
"beneficial": 169,
|
| 451 |
+
"harmful": 3,
|
| 452 |
+
"rewrite_precision": 0.9825581395348837
|
| 453 |
+
},
|
| 454 |
+
{
|
| 455 |
+
"threshold": 0.66,
|
| 456 |
+
"samples": 467,
|
| 457 |
+
"teacher_exact": 0.3576017130620985,
|
| 458 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 459 |
+
"final_exact": 0.7130620985010707,
|
| 460 |
+
"gain_pp": 35.54603854389722,
|
| 461 |
+
"rewrites": 338,
|
| 462 |
+
"abstained": 129,
|
| 463 |
+
"beneficial": 169,
|
| 464 |
+
"harmful": 3,
|
| 465 |
+
"rewrite_precision": 0.9825581395348837
|
| 466 |
+
},
|
| 467 |
+
{
|
| 468 |
+
"threshold": 0.68,
|
| 469 |
+
"samples": 467,
|
| 470 |
+
"teacher_exact": 0.3576017130620985,
|
| 471 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 472 |
+
"final_exact": 0.7109207708779444,
|
| 473 |
+
"gain_pp": 35.33190578158458,
|
| 474 |
+
"rewrites": 337,
|
| 475 |
+
"abstained": 130,
|
| 476 |
+
"beneficial": 168,
|
| 477 |
+
"harmful": 3,
|
| 478 |
+
"rewrite_precision": 0.9824561403508771
|
| 479 |
+
},
|
| 480 |
+
{
|
| 481 |
+
"threshold": 0.7,
|
| 482 |
+
"samples": 467,
|
| 483 |
+
"teacher_exact": 0.3576017130620985,
|
| 484 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 485 |
+
"final_exact": 0.7066381156316917,
|
| 486 |
+
"gain_pp": 34.903640256959314,
|
| 487 |
+
"rewrites": 335,
|
| 488 |
+
"abstained": 132,
|
| 489 |
+
"beneficial": 166,
|
| 490 |
+
"harmful": 3,
|
| 491 |
+
"rewrite_precision": 0.9822485207100592
|
| 492 |
+
},
|
| 493 |
+
{
|
| 494 |
+
"threshold": 0.72,
|
| 495 |
+
"samples": 467,
|
| 496 |
+
"teacher_exact": 0.3576017130620985,
|
| 497 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 498 |
+
"final_exact": 0.7066381156316917,
|
| 499 |
+
"gain_pp": 34.903640256959314,
|
| 500 |
+
"rewrites": 335,
|
| 501 |
+
"abstained": 132,
|
| 502 |
+
"beneficial": 166,
|
| 503 |
+
"harmful": 3,
|
| 504 |
+
"rewrite_precision": 0.9822485207100592
|
| 505 |
+
},
|
| 506 |
+
{
|
| 507 |
+
"threshold": 0.74,
|
| 508 |
+
"samples": 467,
|
| 509 |
+
"teacher_exact": 0.3576017130620985,
|
| 510 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 511 |
+
"final_exact": 0.7066381156316917,
|
| 512 |
+
"gain_pp": 34.903640256959314,
|
| 513 |
+
"rewrites": 335,
|
| 514 |
+
"abstained": 132,
|
| 515 |
+
"beneficial": 166,
|
| 516 |
+
"harmful": 3,
|
| 517 |
+
"rewrite_precision": 0.9822485207100592
|
| 518 |
+
},
|
| 519 |
+
{
|
| 520 |
+
"threshold": 0.76,
|
| 521 |
+
"samples": 467,
|
| 522 |
+
"teacher_exact": 0.3576017130620985,
|
| 523 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 524 |
+
"final_exact": 0.7066381156316917,
|
| 525 |
+
"gain_pp": 34.903640256959314,
|
| 526 |
+
"rewrites": 335,
|
| 527 |
+
"abstained": 132,
|
| 528 |
+
"beneficial": 166,
|
| 529 |
+
"harmful": 3,
|
| 530 |
+
"rewrite_precision": 0.9822485207100592
|
| 531 |
+
},
|
| 532 |
+
{
|
| 533 |
+
"threshold": 0.78,
|
| 534 |
+
"samples": 467,
|
| 535 |
+
"teacher_exact": 0.3576017130620985,
|
| 536 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 537 |
+
"final_exact": 0.7066381156316917,
|
| 538 |
+
"gain_pp": 34.903640256959314,
|
| 539 |
+
"rewrites": 334,
|
| 540 |
+
"abstained": 133,
|
| 541 |
+
"beneficial": 166,
|
| 542 |
+
"harmful": 3,
|
| 543 |
+
"rewrite_precision": 0.9822485207100592
|
| 544 |
+
},
|
| 545 |
+
{
|
| 546 |
+
"threshold": 0.8,
|
| 547 |
+
"samples": 467,
|
| 548 |
+
"teacher_exact": 0.3576017130620985,
|
| 549 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 550 |
+
"final_exact": 0.7066381156316917,
|
| 551 |
+
"gain_pp": 34.903640256959314,
|
| 552 |
+
"rewrites": 334,
|
| 553 |
+
"abstained": 133,
|
| 554 |
+
"beneficial": 166,
|
| 555 |
+
"harmful": 3,
|
| 556 |
+
"rewrite_precision": 0.9822485207100592
|
| 557 |
+
},
|
| 558 |
+
{
|
| 559 |
+
"threshold": 0.82,
|
| 560 |
+
"samples": 467,
|
| 561 |
+
"teacher_exact": 0.3576017130620985,
|
| 562 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 563 |
+
"final_exact": 0.7066381156316917,
|
| 564 |
+
"gain_pp": 34.903640256959314,
|
| 565 |
+
"rewrites": 332,
|
| 566 |
+
"abstained": 135,
|
| 567 |
+
"beneficial": 165,
|
| 568 |
+
"harmful": 2,
|
| 569 |
+
"rewrite_precision": 0.9880239520958084
|
| 570 |
+
},
|
| 571 |
+
{
|
| 572 |
+
"threshold": 0.84,
|
| 573 |
+
"samples": 467,
|
| 574 |
+
"teacher_exact": 0.3576017130620985,
|
| 575 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 576 |
+
"final_exact": 0.7066381156316917,
|
| 577 |
+
"gain_pp": 34.903640256959314,
|
| 578 |
+
"rewrites": 330,
|
| 579 |
+
"abstained": 137,
|
| 580 |
+
"beneficial": 164,
|
| 581 |
+
"harmful": 1,
|
| 582 |
+
"rewrite_precision": 0.9939393939393939
|
| 583 |
+
},
|
| 584 |
+
{
|
| 585 |
+
"threshold": 0.86,
|
| 586 |
+
"samples": 467,
|
| 587 |
+
"teacher_exact": 0.3576017130620985,
|
| 588 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 589 |
+
"final_exact": 0.6980728051391863,
|
| 590 |
+
"gain_pp": 34.04710920770878,
|
| 591 |
+
"rewrites": 326,
|
| 592 |
+
"abstained": 141,
|
| 593 |
+
"beneficial": 160,
|
| 594 |
+
"harmful": 1,
|
| 595 |
+
"rewrite_precision": 0.9937888198757764
|
| 596 |
+
},
|
| 597 |
+
{
|
| 598 |
+
"threshold": 0.88,
|
| 599 |
+
"samples": 467,
|
| 600 |
+
"teacher_exact": 0.3576017130620985,
|
| 601 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 602 |
+
"final_exact": 0.69593147751606,
|
| 603 |
+
"gain_pp": 33.83297644539615,
|
| 604 |
+
"rewrites": 322,
|
| 605 |
+
"abstained": 145,
|
| 606 |
+
"beneficial": 158,
|
| 607 |
+
"harmful": 0,
|
| 608 |
+
"rewrite_precision": 1.0
|
| 609 |
+
},
|
| 610 |
+
{
|
| 611 |
+
"threshold": 0.9,
|
| 612 |
+
"samples": 467,
|
| 613 |
+
"teacher_exact": 0.3576017130620985,
|
| 614 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 615 |
+
"final_exact": 0.6937901498929336,
|
| 616 |
+
"gain_pp": 33.61884368308351,
|
| 617 |
+
"rewrites": 319,
|
| 618 |
+
"abstained": 148,
|
| 619 |
+
"beneficial": 157,
|
| 620 |
+
"harmful": 0,
|
| 621 |
+
"rewrite_precision": 1.0
|
| 622 |
+
},
|
| 623 |
+
{
|
| 624 |
+
"threshold": 0.92,
|
| 625 |
+
"samples": 467,
|
| 626 |
+
"teacher_exact": 0.3576017130620985,
|
| 627 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 628 |
+
"final_exact": 0.6895074946466809,
|
| 629 |
+
"gain_pp": 33.190578158458244,
|
| 630 |
+
"rewrites": 317,
|
| 631 |
+
"abstained": 150,
|
| 632 |
+
"beneficial": 155,
|
| 633 |
+
"harmful": 0,
|
| 634 |
+
"rewrite_precision": 1.0
|
| 635 |
+
},
|
| 636 |
+
{
|
| 637 |
+
"threshold": 0.94,
|
| 638 |
+
"samples": 467,
|
| 639 |
+
"teacher_exact": 0.3576017130620985,
|
| 640 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 641 |
+
"final_exact": 0.6873661670235546,
|
| 642 |
+
"gain_pp": 32.97644539614561,
|
| 643 |
+
"rewrites": 315,
|
| 644 |
+
"abstained": 152,
|
| 645 |
+
"beneficial": 154,
|
| 646 |
+
"harmful": 0,
|
| 647 |
+
"rewrite_precision": 1.0
|
| 648 |
+
},
|
| 649 |
+
{
|
| 650 |
+
"threshold": 0.96,
|
| 651 |
+
"samples": 467,
|
| 652 |
+
"teacher_exact": 0.3576017130620985,
|
| 653 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 654 |
+
"final_exact": 0.6873661670235546,
|
| 655 |
+
"gain_pp": 32.97644539614561,
|
| 656 |
+
"rewrites": 313,
|
| 657 |
+
"abstained": 154,
|
| 658 |
+
"beneficial": 154,
|
| 659 |
+
"harmful": 0,
|
| 660 |
+
"rewrite_precision": 1.0
|
| 661 |
+
},
|
| 662 |
+
{
|
| 663 |
+
"threshold": 0.98,
|
| 664 |
+
"samples": 467,
|
| 665 |
+
"teacher_exact": 0.3576017130620985,
|
| 666 |
+
"teacher_visual_family": 0.7301927194860813,
|
| 667 |
+
"final_exact": 0.6638115631691649,
|
| 668 |
+
"gain_pp": 30.620985010706637,
|
| 669 |
+
"rewrites": 293,
|
| 670 |
+
"abstained": 174,
|
| 671 |
+
"beneficial": 143,
|
| 672 |
+
"harmful": 0,
|
| 673 |
+
"rewrite_precision": 1.0
|
| 674 |
+
}
|
| 675 |
+
],
|
| 676 |
+
"official_test": {
|
| 677 |
+
"threshold": 0.54,
|
| 678 |
+
"samples": 635,
|
| 679 |
+
"teacher_exact": 0.4519685039370079,
|
| 680 |
+
"teacher_visual_family": 0.6425196850393701,
|
| 681 |
+
"final_exact": 0.584251968503937,
|
| 682 |
+
"gain_pp": 13.228346456692913,
|
| 683 |
+
"rewrites": 399,
|
| 684 |
+
"abstained": 236,
|
| 685 |
+
"beneficial": 108,
|
| 686 |
+
"harmful": 24,
|
| 687 |
+
"rewrite_precision": 0.8181818181818182
|
| 688 |
+
}
|
| 689 |
+
},
|
| 690 |
+
{
|
| 691 |
+
"adapter": "research\\runs\\math_ink_06_online_casecontext_refined_seed31_20260723\\skeleton_adapter.pt",
|
| 692 |
+
"behavior_head": "research\\runs\\math_ink_06_behavior_role_seed31_20260724\\behavior_role_head.pt",
|
| 693 |
+
"selected_threshold": 0.9,
|
| 694 |
+
"validation_selected": {
|
| 695 |
+
"threshold": 0.9,
|
| 696 |
+
"samples": 467,
|
| 697 |
+
"teacher_exact": 0.1905781584582441,
|
| 698 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 699 |
+
"final_exact": 0.5438972162740899,
|
| 700 |
+
"gain_pp": 35.33190578158458,
|
| 701 |
+
"rewrites": 250,
|
| 702 |
+
"abstained": 217,
|
| 703 |
+
"beneficial": 165,
|
| 704 |
+
"harmful": 0,
|
| 705 |
+
"rewrite_precision": 1.0
|
| 706 |
+
},
|
| 707 |
+
"validation_sweep": [
|
| 708 |
+
{
|
| 709 |
+
"threshold": 0.0,
|
| 710 |
+
"samples": 467,
|
| 711 |
+
"teacher_exact": 0.1905781584582441,
|
| 712 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 713 |
+
"final_exact": 0.5417558886509636,
|
| 714 |
+
"gain_pp": 35.11777301927195,
|
| 715 |
+
"rewrites": 256,
|
| 716 |
+
"abstained": 211,
|
| 717 |
+
"beneficial": 166,
|
| 718 |
+
"harmful": 2,
|
| 719 |
+
"rewrite_precision": 0.9880952380952381
|
| 720 |
+
},
|
| 721 |
+
{
|
| 722 |
+
"threshold": 0.02,
|
| 723 |
+
"samples": 467,
|
| 724 |
+
"teacher_exact": 0.1905781584582441,
|
| 725 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 726 |
+
"final_exact": 0.5417558886509636,
|
| 727 |
+
"gain_pp": 35.11777301927195,
|
| 728 |
+
"rewrites": 256,
|
| 729 |
+
"abstained": 211,
|
| 730 |
+
"beneficial": 166,
|
| 731 |
+
"harmful": 2,
|
| 732 |
+
"rewrite_precision": 0.9880952380952381
|
| 733 |
+
},
|
| 734 |
+
{
|
| 735 |
+
"threshold": 0.04,
|
| 736 |
+
"samples": 467,
|
| 737 |
+
"teacher_exact": 0.1905781584582441,
|
| 738 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 739 |
+
"final_exact": 0.5417558886509636,
|
| 740 |
+
"gain_pp": 35.11777301927195,
|
| 741 |
+
"rewrites": 256,
|
| 742 |
+
"abstained": 211,
|
| 743 |
+
"beneficial": 166,
|
| 744 |
+
"harmful": 2,
|
| 745 |
+
"rewrite_precision": 0.9880952380952381
|
| 746 |
+
},
|
| 747 |
+
{
|
| 748 |
+
"threshold": 0.06,
|
| 749 |
+
"samples": 467,
|
| 750 |
+
"teacher_exact": 0.1905781584582441,
|
| 751 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 752 |
+
"final_exact": 0.5417558886509636,
|
| 753 |
+
"gain_pp": 35.11777301927195,
|
| 754 |
+
"rewrites": 256,
|
| 755 |
+
"abstained": 211,
|
| 756 |
+
"beneficial": 166,
|
| 757 |
+
"harmful": 2,
|
| 758 |
+
"rewrite_precision": 0.9880952380952381
|
| 759 |
+
},
|
| 760 |
+
{
|
| 761 |
+
"threshold": 0.08,
|
| 762 |
+
"samples": 467,
|
| 763 |
+
"teacher_exact": 0.1905781584582441,
|
| 764 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 765 |
+
"final_exact": 0.5417558886509636,
|
| 766 |
+
"gain_pp": 35.11777301927195,
|
| 767 |
+
"rewrites": 256,
|
| 768 |
+
"abstained": 211,
|
| 769 |
+
"beneficial": 166,
|
| 770 |
+
"harmful": 2,
|
| 771 |
+
"rewrite_precision": 0.9880952380952381
|
| 772 |
+
},
|
| 773 |
+
{
|
| 774 |
+
"threshold": 0.1,
|
| 775 |
+
"samples": 467,
|
| 776 |
+
"teacher_exact": 0.1905781584582441,
|
| 777 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 778 |
+
"final_exact": 0.5417558886509636,
|
| 779 |
+
"gain_pp": 35.11777301927195,
|
| 780 |
+
"rewrites": 256,
|
| 781 |
+
"abstained": 211,
|
| 782 |
+
"beneficial": 166,
|
| 783 |
+
"harmful": 2,
|
| 784 |
+
"rewrite_precision": 0.9880952380952381
|
| 785 |
+
},
|
| 786 |
+
{
|
| 787 |
+
"threshold": 0.12,
|
| 788 |
+
"samples": 467,
|
| 789 |
+
"teacher_exact": 0.1905781584582441,
|
| 790 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 791 |
+
"final_exact": 0.5417558886509636,
|
| 792 |
+
"gain_pp": 35.11777301927195,
|
| 793 |
+
"rewrites": 256,
|
| 794 |
+
"abstained": 211,
|
| 795 |
+
"beneficial": 166,
|
| 796 |
+
"harmful": 2,
|
| 797 |
+
"rewrite_precision": 0.9880952380952381
|
| 798 |
+
},
|
| 799 |
+
{
|
| 800 |
+
"threshold": 0.14,
|
| 801 |
+
"samples": 467,
|
| 802 |
+
"teacher_exact": 0.1905781584582441,
|
| 803 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 804 |
+
"final_exact": 0.5417558886509636,
|
| 805 |
+
"gain_pp": 35.11777301927195,
|
| 806 |
+
"rewrites": 256,
|
| 807 |
+
"abstained": 211,
|
| 808 |
+
"beneficial": 166,
|
| 809 |
+
"harmful": 2,
|
| 810 |
+
"rewrite_precision": 0.9880952380952381
|
| 811 |
+
},
|
| 812 |
+
{
|
| 813 |
+
"threshold": 0.16,
|
| 814 |
+
"samples": 467,
|
| 815 |
+
"teacher_exact": 0.1905781584582441,
|
| 816 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 817 |
+
"final_exact": 0.5417558886509636,
|
| 818 |
+
"gain_pp": 35.11777301927195,
|
| 819 |
+
"rewrites": 256,
|
| 820 |
+
"abstained": 211,
|
| 821 |
+
"beneficial": 166,
|
| 822 |
+
"harmful": 2,
|
| 823 |
+
"rewrite_precision": 0.9880952380952381
|
| 824 |
+
},
|
| 825 |
+
{
|
| 826 |
+
"threshold": 0.18,
|
| 827 |
+
"samples": 467,
|
| 828 |
+
"teacher_exact": 0.1905781584582441,
|
| 829 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 830 |
+
"final_exact": 0.5417558886509636,
|
| 831 |
+
"gain_pp": 35.11777301927195,
|
| 832 |
+
"rewrites": 256,
|
| 833 |
+
"abstained": 211,
|
| 834 |
+
"beneficial": 166,
|
| 835 |
+
"harmful": 2,
|
| 836 |
+
"rewrite_precision": 0.9880952380952381
|
| 837 |
+
},
|
| 838 |
+
{
|
| 839 |
+
"threshold": 0.2,
|
| 840 |
+
"samples": 467,
|
| 841 |
+
"teacher_exact": 0.1905781584582441,
|
| 842 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 843 |
+
"final_exact": 0.5417558886509636,
|
| 844 |
+
"gain_pp": 35.11777301927195,
|
| 845 |
+
"rewrites": 256,
|
| 846 |
+
"abstained": 211,
|
| 847 |
+
"beneficial": 166,
|
| 848 |
+
"harmful": 2,
|
| 849 |
+
"rewrite_precision": 0.9880952380952381
|
| 850 |
+
},
|
| 851 |
+
{
|
| 852 |
+
"threshold": 0.22,
|
| 853 |
+
"samples": 467,
|
| 854 |
+
"teacher_exact": 0.1905781584582441,
|
| 855 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 856 |
+
"final_exact": 0.5417558886509636,
|
| 857 |
+
"gain_pp": 35.11777301927195,
|
| 858 |
+
"rewrites": 256,
|
| 859 |
+
"abstained": 211,
|
| 860 |
+
"beneficial": 166,
|
| 861 |
+
"harmful": 2,
|
| 862 |
+
"rewrite_precision": 0.9880952380952381
|
| 863 |
+
},
|
| 864 |
+
{
|
| 865 |
+
"threshold": 0.24,
|
| 866 |
+
"samples": 467,
|
| 867 |
+
"teacher_exact": 0.1905781584582441,
|
| 868 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 869 |
+
"final_exact": 0.5417558886509636,
|
| 870 |
+
"gain_pp": 35.11777301927195,
|
| 871 |
+
"rewrites": 256,
|
| 872 |
+
"abstained": 211,
|
| 873 |
+
"beneficial": 166,
|
| 874 |
+
"harmful": 2,
|
| 875 |
+
"rewrite_precision": 0.9880952380952381
|
| 876 |
+
},
|
| 877 |
+
{
|
| 878 |
+
"threshold": 0.26,
|
| 879 |
+
"samples": 467,
|
| 880 |
+
"teacher_exact": 0.1905781584582441,
|
| 881 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 882 |
+
"final_exact": 0.5417558886509636,
|
| 883 |
+
"gain_pp": 35.11777301927195,
|
| 884 |
+
"rewrites": 256,
|
| 885 |
+
"abstained": 211,
|
| 886 |
+
"beneficial": 166,
|
| 887 |
+
"harmful": 2,
|
| 888 |
+
"rewrite_precision": 0.9880952380952381
|
| 889 |
+
},
|
| 890 |
+
{
|
| 891 |
+
"threshold": 0.28,
|
| 892 |
+
"samples": 467,
|
| 893 |
+
"teacher_exact": 0.1905781584582441,
|
| 894 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 895 |
+
"final_exact": 0.5417558886509636,
|
| 896 |
+
"gain_pp": 35.11777301927195,
|
| 897 |
+
"rewrites": 256,
|
| 898 |
+
"abstained": 211,
|
| 899 |
+
"beneficial": 166,
|
| 900 |
+
"harmful": 2,
|
| 901 |
+
"rewrite_precision": 0.9880952380952381
|
| 902 |
+
},
|
| 903 |
+
{
|
| 904 |
+
"threshold": 0.3,
|
| 905 |
+
"samples": 467,
|
| 906 |
+
"teacher_exact": 0.1905781584582441,
|
| 907 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 908 |
+
"final_exact": 0.5417558886509636,
|
| 909 |
+
"gain_pp": 35.11777301927195,
|
| 910 |
+
"rewrites": 256,
|
| 911 |
+
"abstained": 211,
|
| 912 |
+
"beneficial": 166,
|
| 913 |
+
"harmful": 2,
|
| 914 |
+
"rewrite_precision": 0.9880952380952381
|
| 915 |
+
},
|
| 916 |
+
{
|
| 917 |
+
"threshold": 0.32,
|
| 918 |
+
"samples": 467,
|
| 919 |
+
"teacher_exact": 0.1905781584582441,
|
| 920 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 921 |
+
"final_exact": 0.5417558886509636,
|
| 922 |
+
"gain_pp": 35.11777301927195,
|
| 923 |
+
"rewrites": 256,
|
| 924 |
+
"abstained": 211,
|
| 925 |
+
"beneficial": 166,
|
| 926 |
+
"harmful": 2,
|
| 927 |
+
"rewrite_precision": 0.9880952380952381
|
| 928 |
+
},
|
| 929 |
+
{
|
| 930 |
+
"threshold": 0.34,
|
| 931 |
+
"samples": 467,
|
| 932 |
+
"teacher_exact": 0.1905781584582441,
|
| 933 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 934 |
+
"final_exact": 0.5417558886509636,
|
| 935 |
+
"gain_pp": 35.11777301927195,
|
| 936 |
+
"rewrites": 256,
|
| 937 |
+
"abstained": 211,
|
| 938 |
+
"beneficial": 166,
|
| 939 |
+
"harmful": 2,
|
| 940 |
+
"rewrite_precision": 0.9880952380952381
|
| 941 |
+
},
|
| 942 |
+
{
|
| 943 |
+
"threshold": 0.36,
|
| 944 |
+
"samples": 467,
|
| 945 |
+
"teacher_exact": 0.1905781584582441,
|
| 946 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 947 |
+
"final_exact": 0.5417558886509636,
|
| 948 |
+
"gain_pp": 35.11777301927195,
|
| 949 |
+
"rewrites": 256,
|
| 950 |
+
"abstained": 211,
|
| 951 |
+
"beneficial": 166,
|
| 952 |
+
"harmful": 2,
|
| 953 |
+
"rewrite_precision": 0.9880952380952381
|
| 954 |
+
},
|
| 955 |
+
{
|
| 956 |
+
"threshold": 0.38,
|
| 957 |
+
"samples": 467,
|
| 958 |
+
"teacher_exact": 0.1905781584582441,
|
| 959 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 960 |
+
"final_exact": 0.5417558886509636,
|
| 961 |
+
"gain_pp": 35.11777301927195,
|
| 962 |
+
"rewrites": 256,
|
| 963 |
+
"abstained": 211,
|
| 964 |
+
"beneficial": 166,
|
| 965 |
+
"harmful": 2,
|
| 966 |
+
"rewrite_precision": 0.9880952380952381
|
| 967 |
+
},
|
| 968 |
+
{
|
| 969 |
+
"threshold": 0.4,
|
| 970 |
+
"samples": 467,
|
| 971 |
+
"teacher_exact": 0.1905781584582441,
|
| 972 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 973 |
+
"final_exact": 0.5417558886509636,
|
| 974 |
+
"gain_pp": 35.11777301927195,
|
| 975 |
+
"rewrites": 256,
|
| 976 |
+
"abstained": 211,
|
| 977 |
+
"beneficial": 166,
|
| 978 |
+
"harmful": 2,
|
| 979 |
+
"rewrite_precision": 0.9880952380952381
|
| 980 |
+
},
|
| 981 |
+
{
|
| 982 |
+
"threshold": 0.42,
|
| 983 |
+
"samples": 467,
|
| 984 |
+
"teacher_exact": 0.1905781584582441,
|
| 985 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 986 |
+
"final_exact": 0.5417558886509636,
|
| 987 |
+
"gain_pp": 35.11777301927195,
|
| 988 |
+
"rewrites": 256,
|
| 989 |
+
"abstained": 211,
|
| 990 |
+
"beneficial": 166,
|
| 991 |
+
"harmful": 2,
|
| 992 |
+
"rewrite_precision": 0.9880952380952381
|
| 993 |
+
},
|
| 994 |
+
{
|
| 995 |
+
"threshold": 0.44,
|
| 996 |
+
"samples": 467,
|
| 997 |
+
"teacher_exact": 0.1905781584582441,
|
| 998 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 999 |
+
"final_exact": 0.5417558886509636,
|
| 1000 |
+
"gain_pp": 35.11777301927195,
|
| 1001 |
+
"rewrites": 256,
|
| 1002 |
+
"abstained": 211,
|
| 1003 |
+
"beneficial": 166,
|
| 1004 |
+
"harmful": 2,
|
| 1005 |
+
"rewrite_precision": 0.9880952380952381
|
| 1006 |
+
},
|
| 1007 |
+
{
|
| 1008 |
+
"threshold": 0.46,
|
| 1009 |
+
"samples": 467,
|
| 1010 |
+
"teacher_exact": 0.1905781584582441,
|
| 1011 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1012 |
+
"final_exact": 0.5417558886509636,
|
| 1013 |
+
"gain_pp": 35.11777301927195,
|
| 1014 |
+
"rewrites": 256,
|
| 1015 |
+
"abstained": 211,
|
| 1016 |
+
"beneficial": 166,
|
| 1017 |
+
"harmful": 2,
|
| 1018 |
+
"rewrite_precision": 0.9880952380952381
|
| 1019 |
+
},
|
| 1020 |
+
{
|
| 1021 |
+
"threshold": 0.48,
|
| 1022 |
+
"samples": 467,
|
| 1023 |
+
"teacher_exact": 0.1905781584582441,
|
| 1024 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1025 |
+
"final_exact": 0.5417558886509636,
|
| 1026 |
+
"gain_pp": 35.11777301927195,
|
| 1027 |
+
"rewrites": 256,
|
| 1028 |
+
"abstained": 211,
|
| 1029 |
+
"beneficial": 166,
|
| 1030 |
+
"harmful": 2,
|
| 1031 |
+
"rewrite_precision": 0.9880952380952381
|
| 1032 |
+
},
|
| 1033 |
+
{
|
| 1034 |
+
"threshold": 0.5,
|
| 1035 |
+
"samples": 467,
|
| 1036 |
+
"teacher_exact": 0.1905781584582441,
|
| 1037 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1038 |
+
"final_exact": 0.5417558886509636,
|
| 1039 |
+
"gain_pp": 35.11777301927195,
|
| 1040 |
+
"rewrites": 255,
|
| 1041 |
+
"abstained": 212,
|
| 1042 |
+
"beneficial": 166,
|
| 1043 |
+
"harmful": 2,
|
| 1044 |
+
"rewrite_precision": 0.9880952380952381
|
| 1045 |
+
},
|
| 1046 |
+
{
|
| 1047 |
+
"threshold": 0.52,
|
| 1048 |
+
"samples": 467,
|
| 1049 |
+
"teacher_exact": 0.1905781584582441,
|
| 1050 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1051 |
+
"final_exact": 0.5417558886509636,
|
| 1052 |
+
"gain_pp": 35.11777301927195,
|
| 1053 |
+
"rewrites": 254,
|
| 1054 |
+
"abstained": 213,
|
| 1055 |
+
"beneficial": 166,
|
| 1056 |
+
"harmful": 2,
|
| 1057 |
+
"rewrite_precision": 0.9880952380952381
|
| 1058 |
+
},
|
| 1059 |
+
{
|
| 1060 |
+
"threshold": 0.54,
|
| 1061 |
+
"samples": 467,
|
| 1062 |
+
"teacher_exact": 0.1905781584582441,
|
| 1063 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1064 |
+
"final_exact": 0.5417558886509636,
|
| 1065 |
+
"gain_pp": 35.11777301927195,
|
| 1066 |
+
"rewrites": 254,
|
| 1067 |
+
"abstained": 213,
|
| 1068 |
+
"beneficial": 166,
|
| 1069 |
+
"harmful": 2,
|
| 1070 |
+
"rewrite_precision": 0.9880952380952381
|
| 1071 |
+
},
|
| 1072 |
+
{
|
| 1073 |
+
"threshold": 0.56,
|
| 1074 |
+
"samples": 467,
|
| 1075 |
+
"teacher_exact": 0.1905781584582441,
|
| 1076 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1077 |
+
"final_exact": 0.5417558886509636,
|
| 1078 |
+
"gain_pp": 35.11777301927195,
|
| 1079 |
+
"rewrites": 254,
|
| 1080 |
+
"abstained": 213,
|
| 1081 |
+
"beneficial": 166,
|
| 1082 |
+
"harmful": 2,
|
| 1083 |
+
"rewrite_precision": 0.9880952380952381
|
| 1084 |
+
},
|
| 1085 |
+
{
|
| 1086 |
+
"threshold": 0.58,
|
| 1087 |
+
"samples": 467,
|
| 1088 |
+
"teacher_exact": 0.1905781584582441,
|
| 1089 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1090 |
+
"final_exact": 0.5417558886509636,
|
| 1091 |
+
"gain_pp": 35.11777301927195,
|
| 1092 |
+
"rewrites": 254,
|
| 1093 |
+
"abstained": 213,
|
| 1094 |
+
"beneficial": 166,
|
| 1095 |
+
"harmful": 2,
|
| 1096 |
+
"rewrite_precision": 0.9880952380952381
|
| 1097 |
+
},
|
| 1098 |
+
{
|
| 1099 |
+
"threshold": 0.6,
|
| 1100 |
+
"samples": 467,
|
| 1101 |
+
"teacher_exact": 0.1905781584582441,
|
| 1102 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1103 |
+
"final_exact": 0.5417558886509636,
|
| 1104 |
+
"gain_pp": 35.11777301927195,
|
| 1105 |
+
"rewrites": 254,
|
| 1106 |
+
"abstained": 213,
|
| 1107 |
+
"beneficial": 166,
|
| 1108 |
+
"harmful": 2,
|
| 1109 |
+
"rewrite_precision": 0.9880952380952381
|
| 1110 |
+
},
|
| 1111 |
+
{
|
| 1112 |
+
"threshold": 0.62,
|
| 1113 |
+
"samples": 467,
|
| 1114 |
+
"teacher_exact": 0.1905781584582441,
|
| 1115 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1116 |
+
"final_exact": 0.5417558886509636,
|
| 1117 |
+
"gain_pp": 35.11777301927195,
|
| 1118 |
+
"rewrites": 254,
|
| 1119 |
+
"abstained": 213,
|
| 1120 |
+
"beneficial": 166,
|
| 1121 |
+
"harmful": 2,
|
| 1122 |
+
"rewrite_precision": 0.9880952380952381
|
| 1123 |
+
},
|
| 1124 |
+
{
|
| 1125 |
+
"threshold": 0.64,
|
| 1126 |
+
"samples": 467,
|
| 1127 |
+
"teacher_exact": 0.1905781584582441,
|
| 1128 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1129 |
+
"final_exact": 0.5417558886509636,
|
| 1130 |
+
"gain_pp": 35.11777301927195,
|
| 1131 |
+
"rewrites": 254,
|
| 1132 |
+
"abstained": 213,
|
| 1133 |
+
"beneficial": 166,
|
| 1134 |
+
"harmful": 2,
|
| 1135 |
+
"rewrite_precision": 0.9880952380952381
|
| 1136 |
+
},
|
| 1137 |
+
{
|
| 1138 |
+
"threshold": 0.66,
|
| 1139 |
+
"samples": 467,
|
| 1140 |
+
"teacher_exact": 0.1905781584582441,
|
| 1141 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1142 |
+
"final_exact": 0.5396145610278372,
|
| 1143 |
+
"gain_pp": 34.903640256959314,
|
| 1144 |
+
"rewrites": 253,
|
| 1145 |
+
"abstained": 214,
|
| 1146 |
+
"beneficial": 165,
|
| 1147 |
+
"harmful": 2,
|
| 1148 |
+
"rewrite_precision": 0.9880239520958084
|
| 1149 |
+
},
|
| 1150 |
+
{
|
| 1151 |
+
"threshold": 0.68,
|
| 1152 |
+
"samples": 467,
|
| 1153 |
+
"teacher_exact": 0.1905781584582441,
|
| 1154 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1155 |
+
"final_exact": 0.5396145610278372,
|
| 1156 |
+
"gain_pp": 34.903640256959314,
|
| 1157 |
+
"rewrites": 253,
|
| 1158 |
+
"abstained": 214,
|
| 1159 |
+
"beneficial": 165,
|
| 1160 |
+
"harmful": 2,
|
| 1161 |
+
"rewrite_precision": 0.9880239520958084
|
| 1162 |
+
},
|
| 1163 |
+
{
|
| 1164 |
+
"threshold": 0.7,
|
| 1165 |
+
"samples": 467,
|
| 1166 |
+
"teacher_exact": 0.1905781584582441,
|
| 1167 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1168 |
+
"final_exact": 0.5396145610278372,
|
| 1169 |
+
"gain_pp": 34.903640256959314,
|
| 1170 |
+
"rewrites": 252,
|
| 1171 |
+
"abstained": 215,
|
| 1172 |
+
"beneficial": 165,
|
| 1173 |
+
"harmful": 2,
|
| 1174 |
+
"rewrite_precision": 0.9880239520958084
|
| 1175 |
+
},
|
| 1176 |
+
{
|
| 1177 |
+
"threshold": 0.72,
|
| 1178 |
+
"samples": 467,
|
| 1179 |
+
"teacher_exact": 0.1905781584582441,
|
| 1180 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1181 |
+
"final_exact": 0.5417558886509636,
|
| 1182 |
+
"gain_pp": 35.11777301927195,
|
| 1183 |
+
"rewrites": 251,
|
| 1184 |
+
"abstained": 216,
|
| 1185 |
+
"beneficial": 165,
|
| 1186 |
+
"harmful": 1,
|
| 1187 |
+
"rewrite_precision": 0.9939759036144579
|
| 1188 |
+
},
|
| 1189 |
+
{
|
| 1190 |
+
"threshold": 0.74,
|
| 1191 |
+
"samples": 467,
|
| 1192 |
+
"teacher_exact": 0.1905781584582441,
|
| 1193 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1194 |
+
"final_exact": 0.5417558886509636,
|
| 1195 |
+
"gain_pp": 35.11777301927195,
|
| 1196 |
+
"rewrites": 251,
|
| 1197 |
+
"abstained": 216,
|
| 1198 |
+
"beneficial": 165,
|
| 1199 |
+
"harmful": 1,
|
| 1200 |
+
"rewrite_precision": 0.9939759036144579
|
| 1201 |
+
},
|
| 1202 |
+
{
|
| 1203 |
+
"threshold": 0.76,
|
| 1204 |
+
"samples": 467,
|
| 1205 |
+
"teacher_exact": 0.1905781584582441,
|
| 1206 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1207 |
+
"final_exact": 0.5417558886509636,
|
| 1208 |
+
"gain_pp": 35.11777301927195,
|
| 1209 |
+
"rewrites": 251,
|
| 1210 |
+
"abstained": 216,
|
| 1211 |
+
"beneficial": 165,
|
| 1212 |
+
"harmful": 1,
|
| 1213 |
+
"rewrite_precision": 0.9939759036144579
|
| 1214 |
+
},
|
| 1215 |
+
{
|
| 1216 |
+
"threshold": 0.78,
|
| 1217 |
+
"samples": 467,
|
| 1218 |
+
"teacher_exact": 0.1905781584582441,
|
| 1219 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1220 |
+
"final_exact": 0.5417558886509636,
|
| 1221 |
+
"gain_pp": 35.11777301927195,
|
| 1222 |
+
"rewrites": 251,
|
| 1223 |
+
"abstained": 216,
|
| 1224 |
+
"beneficial": 165,
|
| 1225 |
+
"harmful": 1,
|
| 1226 |
+
"rewrite_precision": 0.9939759036144579
|
| 1227 |
+
},
|
| 1228 |
+
{
|
| 1229 |
+
"threshold": 0.8,
|
| 1230 |
+
"samples": 467,
|
| 1231 |
+
"teacher_exact": 0.1905781584582441,
|
| 1232 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1233 |
+
"final_exact": 0.5417558886509636,
|
| 1234 |
+
"gain_pp": 35.11777301927195,
|
| 1235 |
+
"rewrites": 251,
|
| 1236 |
+
"abstained": 216,
|
| 1237 |
+
"beneficial": 165,
|
| 1238 |
+
"harmful": 1,
|
| 1239 |
+
"rewrite_precision": 0.9939759036144579
|
| 1240 |
+
},
|
| 1241 |
+
{
|
| 1242 |
+
"threshold": 0.82,
|
| 1243 |
+
"samples": 467,
|
| 1244 |
+
"teacher_exact": 0.1905781584582441,
|
| 1245 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1246 |
+
"final_exact": 0.5438972162740899,
|
| 1247 |
+
"gain_pp": 35.33190578158458,
|
| 1248 |
+
"rewrites": 250,
|
| 1249 |
+
"abstained": 217,
|
| 1250 |
+
"beneficial": 165,
|
| 1251 |
+
"harmful": 0,
|
| 1252 |
+
"rewrite_precision": 1.0
|
| 1253 |
+
},
|
| 1254 |
+
{
|
| 1255 |
+
"threshold": 0.84,
|
| 1256 |
+
"samples": 467,
|
| 1257 |
+
"teacher_exact": 0.1905781584582441,
|
| 1258 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1259 |
+
"final_exact": 0.5438972162740899,
|
| 1260 |
+
"gain_pp": 35.33190578158458,
|
| 1261 |
+
"rewrites": 250,
|
| 1262 |
+
"abstained": 217,
|
| 1263 |
+
"beneficial": 165,
|
| 1264 |
+
"harmful": 0,
|
| 1265 |
+
"rewrite_precision": 1.0
|
| 1266 |
+
},
|
| 1267 |
+
{
|
| 1268 |
+
"threshold": 0.86,
|
| 1269 |
+
"samples": 467,
|
| 1270 |
+
"teacher_exact": 0.1905781584582441,
|
| 1271 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1272 |
+
"final_exact": 0.5438972162740899,
|
| 1273 |
+
"gain_pp": 35.33190578158458,
|
| 1274 |
+
"rewrites": 250,
|
| 1275 |
+
"abstained": 217,
|
| 1276 |
+
"beneficial": 165,
|
| 1277 |
+
"harmful": 0,
|
| 1278 |
+
"rewrite_precision": 1.0
|
| 1279 |
+
},
|
| 1280 |
+
{
|
| 1281 |
+
"threshold": 0.88,
|
| 1282 |
+
"samples": 467,
|
| 1283 |
+
"teacher_exact": 0.1905781584582441,
|
| 1284 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1285 |
+
"final_exact": 0.5438972162740899,
|
| 1286 |
+
"gain_pp": 35.33190578158458,
|
| 1287 |
+
"rewrites": 250,
|
| 1288 |
+
"abstained": 217,
|
| 1289 |
+
"beneficial": 165,
|
| 1290 |
+
"harmful": 0,
|
| 1291 |
+
"rewrite_precision": 1.0
|
| 1292 |
+
},
|
| 1293 |
+
{
|
| 1294 |
+
"threshold": 0.9,
|
| 1295 |
+
"samples": 467,
|
| 1296 |
+
"teacher_exact": 0.1905781584582441,
|
| 1297 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1298 |
+
"final_exact": 0.5438972162740899,
|
| 1299 |
+
"gain_pp": 35.33190578158458,
|
| 1300 |
+
"rewrites": 250,
|
| 1301 |
+
"abstained": 217,
|
| 1302 |
+
"beneficial": 165,
|
| 1303 |
+
"harmful": 0,
|
| 1304 |
+
"rewrite_precision": 1.0
|
| 1305 |
+
},
|
| 1306 |
+
{
|
| 1307 |
+
"threshold": 0.92,
|
| 1308 |
+
"samples": 467,
|
| 1309 |
+
"teacher_exact": 0.1905781584582441,
|
| 1310 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1311 |
+
"final_exact": 0.5417558886509636,
|
| 1312 |
+
"gain_pp": 35.11777301927195,
|
| 1313 |
+
"rewrites": 249,
|
| 1314 |
+
"abstained": 218,
|
| 1315 |
+
"beneficial": 164,
|
| 1316 |
+
"harmful": 0,
|
| 1317 |
+
"rewrite_precision": 1.0
|
| 1318 |
+
},
|
| 1319 |
+
{
|
| 1320 |
+
"threshold": 0.94,
|
| 1321 |
+
"samples": 467,
|
| 1322 |
+
"teacher_exact": 0.1905781584582441,
|
| 1323 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1324 |
+
"final_exact": 0.5396145610278372,
|
| 1325 |
+
"gain_pp": 34.903640256959314,
|
| 1326 |
+
"rewrites": 246,
|
| 1327 |
+
"abstained": 221,
|
| 1328 |
+
"beneficial": 163,
|
| 1329 |
+
"harmful": 0,
|
| 1330 |
+
"rewrite_precision": 1.0
|
| 1331 |
+
},
|
| 1332 |
+
{
|
| 1333 |
+
"threshold": 0.96,
|
| 1334 |
+
"samples": 467,
|
| 1335 |
+
"teacher_exact": 0.1905781584582441,
|
| 1336 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1337 |
+
"final_exact": 0.5396145610278372,
|
| 1338 |
+
"gain_pp": 34.903640256959314,
|
| 1339 |
+
"rewrites": 246,
|
| 1340 |
+
"abstained": 221,
|
| 1341 |
+
"beneficial": 163,
|
| 1342 |
+
"harmful": 0,
|
| 1343 |
+
"rewrite_precision": 1.0
|
| 1344 |
+
},
|
| 1345 |
+
{
|
| 1346 |
+
"threshold": 0.98,
|
| 1347 |
+
"samples": 467,
|
| 1348 |
+
"teacher_exact": 0.1905781584582441,
|
| 1349 |
+
"teacher_visual_family": 0.5481798715203426,
|
| 1350 |
+
"final_exact": 0.5289079229122056,
|
| 1351 |
+
"gain_pp": 33.83297644539615,
|
| 1352 |
+
"rewrites": 238,
|
| 1353 |
+
"abstained": 229,
|
| 1354 |
+
"beneficial": 158,
|
| 1355 |
+
"harmful": 0,
|
| 1356 |
+
"rewrite_precision": 1.0
|
| 1357 |
+
}
|
| 1358 |
+
],
|
| 1359 |
+
"official_test": {
|
| 1360 |
+
"threshold": 0.9,
|
| 1361 |
+
"samples": 635,
|
| 1362 |
+
"teacher_exact": 0.27086614173228346,
|
| 1363 |
+
"teacher_visual_family": 0.44724409448818897,
|
| 1364 |
+
"final_exact": 0.3937007874015748,
|
| 1365 |
+
"gain_pp": 12.283464566929133,
|
| 1366 |
+
"rewrites": 270,
|
| 1367 |
+
"abstained": 365,
|
| 1368 |
+
"beneficial": 104,
|
| 1369 |
+
"harmful": 26,
|
| 1370 |
+
"rewrite_precision": 0.8
|
| 1371 |
+
}
|
| 1372 |
+
},
|
| 1373 |
+
{
|
| 1374 |
+
"adapter": "research\\runs\\math_ink_06_online_casecontext_refined_seed47_20260723\\skeleton_adapter.pt",
|
| 1375 |
+
"behavior_head": "research\\runs\\math_ink_06_behavior_role_seed47_20260724\\behavior_role_head.pt",
|
| 1376 |
+
"selected_threshold": 0.72,
|
| 1377 |
+
"validation_selected": {
|
| 1378 |
+
"threshold": 0.72,
|
| 1379 |
+
"samples": 467,
|
| 1380 |
+
"teacher_exact": 0.24411134903640258,
|
| 1381 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1382 |
+
"final_exact": 0.5867237687366167,
|
| 1383 |
+
"gain_pp": 34.261241970021416,
|
| 1384 |
+
"rewrites": 272,
|
| 1385 |
+
"abstained": 195,
|
| 1386 |
+
"beneficial": 161,
|
| 1387 |
+
"harmful": 1,
|
| 1388 |
+
"rewrite_precision": 0.9938271604938271
|
| 1389 |
+
},
|
| 1390 |
+
"validation_sweep": [
|
| 1391 |
+
{
|
| 1392 |
+
"threshold": 0.0,
|
| 1393 |
+
"samples": 467,
|
| 1394 |
+
"teacher_exact": 0.24411134903640258,
|
| 1395 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1396 |
+
"final_exact": 0.5867237687366167,
|
| 1397 |
+
"gain_pp": 34.261241970021416,
|
| 1398 |
+
"rewrites": 278,
|
| 1399 |
+
"abstained": 189,
|
| 1400 |
+
"beneficial": 162,
|
| 1401 |
+
"harmful": 2,
|
| 1402 |
+
"rewrite_precision": 0.9878048780487805
|
| 1403 |
+
},
|
| 1404 |
+
{
|
| 1405 |
+
"threshold": 0.02,
|
| 1406 |
+
"samples": 467,
|
| 1407 |
+
"teacher_exact": 0.24411134903640258,
|
| 1408 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1409 |
+
"final_exact": 0.5867237687366167,
|
| 1410 |
+
"gain_pp": 34.261241970021416,
|
| 1411 |
+
"rewrites": 278,
|
| 1412 |
+
"abstained": 189,
|
| 1413 |
+
"beneficial": 162,
|
| 1414 |
+
"harmful": 2,
|
| 1415 |
+
"rewrite_precision": 0.9878048780487805
|
| 1416 |
+
},
|
| 1417 |
+
{
|
| 1418 |
+
"threshold": 0.04,
|
| 1419 |
+
"samples": 467,
|
| 1420 |
+
"teacher_exact": 0.24411134903640258,
|
| 1421 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1422 |
+
"final_exact": 0.5867237687366167,
|
| 1423 |
+
"gain_pp": 34.261241970021416,
|
| 1424 |
+
"rewrites": 278,
|
| 1425 |
+
"abstained": 189,
|
| 1426 |
+
"beneficial": 162,
|
| 1427 |
+
"harmful": 2,
|
| 1428 |
+
"rewrite_precision": 0.9878048780487805
|
| 1429 |
+
},
|
| 1430 |
+
{
|
| 1431 |
+
"threshold": 0.06,
|
| 1432 |
+
"samples": 467,
|
| 1433 |
+
"teacher_exact": 0.24411134903640258,
|
| 1434 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1435 |
+
"final_exact": 0.5867237687366167,
|
| 1436 |
+
"gain_pp": 34.261241970021416,
|
| 1437 |
+
"rewrites": 278,
|
| 1438 |
+
"abstained": 189,
|
| 1439 |
+
"beneficial": 162,
|
| 1440 |
+
"harmful": 2,
|
| 1441 |
+
"rewrite_precision": 0.9878048780487805
|
| 1442 |
+
},
|
| 1443 |
+
{
|
| 1444 |
+
"threshold": 0.08,
|
| 1445 |
+
"samples": 467,
|
| 1446 |
+
"teacher_exact": 0.24411134903640258,
|
| 1447 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1448 |
+
"final_exact": 0.5867237687366167,
|
| 1449 |
+
"gain_pp": 34.261241970021416,
|
| 1450 |
+
"rewrites": 278,
|
| 1451 |
+
"abstained": 189,
|
| 1452 |
+
"beneficial": 162,
|
| 1453 |
+
"harmful": 2,
|
| 1454 |
+
"rewrite_precision": 0.9878048780487805
|
| 1455 |
+
},
|
| 1456 |
+
{
|
| 1457 |
+
"threshold": 0.1,
|
| 1458 |
+
"samples": 467,
|
| 1459 |
+
"teacher_exact": 0.24411134903640258,
|
| 1460 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1461 |
+
"final_exact": 0.5867237687366167,
|
| 1462 |
+
"gain_pp": 34.261241970021416,
|
| 1463 |
+
"rewrites": 278,
|
| 1464 |
+
"abstained": 189,
|
| 1465 |
+
"beneficial": 162,
|
| 1466 |
+
"harmful": 2,
|
| 1467 |
+
"rewrite_precision": 0.9878048780487805
|
| 1468 |
+
},
|
| 1469 |
+
{
|
| 1470 |
+
"threshold": 0.12,
|
| 1471 |
+
"samples": 467,
|
| 1472 |
+
"teacher_exact": 0.24411134903640258,
|
| 1473 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1474 |
+
"final_exact": 0.5867237687366167,
|
| 1475 |
+
"gain_pp": 34.261241970021416,
|
| 1476 |
+
"rewrites": 278,
|
| 1477 |
+
"abstained": 189,
|
| 1478 |
+
"beneficial": 162,
|
| 1479 |
+
"harmful": 2,
|
| 1480 |
+
"rewrite_precision": 0.9878048780487805
|
| 1481 |
+
},
|
| 1482 |
+
{
|
| 1483 |
+
"threshold": 0.14,
|
| 1484 |
+
"samples": 467,
|
| 1485 |
+
"teacher_exact": 0.24411134903640258,
|
| 1486 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1487 |
+
"final_exact": 0.5867237687366167,
|
| 1488 |
+
"gain_pp": 34.261241970021416,
|
| 1489 |
+
"rewrites": 278,
|
| 1490 |
+
"abstained": 189,
|
| 1491 |
+
"beneficial": 162,
|
| 1492 |
+
"harmful": 2,
|
| 1493 |
+
"rewrite_precision": 0.9878048780487805
|
| 1494 |
+
},
|
| 1495 |
+
{
|
| 1496 |
+
"threshold": 0.16,
|
| 1497 |
+
"samples": 467,
|
| 1498 |
+
"teacher_exact": 0.24411134903640258,
|
| 1499 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1500 |
+
"final_exact": 0.5867237687366167,
|
| 1501 |
+
"gain_pp": 34.261241970021416,
|
| 1502 |
+
"rewrites": 278,
|
| 1503 |
+
"abstained": 189,
|
| 1504 |
+
"beneficial": 162,
|
| 1505 |
+
"harmful": 2,
|
| 1506 |
+
"rewrite_precision": 0.9878048780487805
|
| 1507 |
+
},
|
| 1508 |
+
{
|
| 1509 |
+
"threshold": 0.18,
|
| 1510 |
+
"samples": 467,
|
| 1511 |
+
"teacher_exact": 0.24411134903640258,
|
| 1512 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1513 |
+
"final_exact": 0.5867237687366167,
|
| 1514 |
+
"gain_pp": 34.261241970021416,
|
| 1515 |
+
"rewrites": 278,
|
| 1516 |
+
"abstained": 189,
|
| 1517 |
+
"beneficial": 162,
|
| 1518 |
+
"harmful": 2,
|
| 1519 |
+
"rewrite_precision": 0.9878048780487805
|
| 1520 |
+
},
|
| 1521 |
+
{
|
| 1522 |
+
"threshold": 0.2,
|
| 1523 |
+
"samples": 467,
|
| 1524 |
+
"teacher_exact": 0.24411134903640258,
|
| 1525 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1526 |
+
"final_exact": 0.5867237687366167,
|
| 1527 |
+
"gain_pp": 34.261241970021416,
|
| 1528 |
+
"rewrites": 278,
|
| 1529 |
+
"abstained": 189,
|
| 1530 |
+
"beneficial": 162,
|
| 1531 |
+
"harmful": 2,
|
| 1532 |
+
"rewrite_precision": 0.9878048780487805
|
| 1533 |
+
},
|
| 1534 |
+
{
|
| 1535 |
+
"threshold": 0.22,
|
| 1536 |
+
"samples": 467,
|
| 1537 |
+
"teacher_exact": 0.24411134903640258,
|
| 1538 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1539 |
+
"final_exact": 0.5867237687366167,
|
| 1540 |
+
"gain_pp": 34.261241970021416,
|
| 1541 |
+
"rewrites": 278,
|
| 1542 |
+
"abstained": 189,
|
| 1543 |
+
"beneficial": 162,
|
| 1544 |
+
"harmful": 2,
|
| 1545 |
+
"rewrite_precision": 0.9878048780487805
|
| 1546 |
+
},
|
| 1547 |
+
{
|
| 1548 |
+
"threshold": 0.24,
|
| 1549 |
+
"samples": 467,
|
| 1550 |
+
"teacher_exact": 0.24411134903640258,
|
| 1551 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1552 |
+
"final_exact": 0.5867237687366167,
|
| 1553 |
+
"gain_pp": 34.261241970021416,
|
| 1554 |
+
"rewrites": 278,
|
| 1555 |
+
"abstained": 189,
|
| 1556 |
+
"beneficial": 162,
|
| 1557 |
+
"harmful": 2,
|
| 1558 |
+
"rewrite_precision": 0.9878048780487805
|
| 1559 |
+
},
|
| 1560 |
+
{
|
| 1561 |
+
"threshold": 0.26,
|
| 1562 |
+
"samples": 467,
|
| 1563 |
+
"teacher_exact": 0.24411134903640258,
|
| 1564 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1565 |
+
"final_exact": 0.5867237687366167,
|
| 1566 |
+
"gain_pp": 34.261241970021416,
|
| 1567 |
+
"rewrites": 278,
|
| 1568 |
+
"abstained": 189,
|
| 1569 |
+
"beneficial": 162,
|
| 1570 |
+
"harmful": 2,
|
| 1571 |
+
"rewrite_precision": 0.9878048780487805
|
| 1572 |
+
},
|
| 1573 |
+
{
|
| 1574 |
+
"threshold": 0.28,
|
| 1575 |
+
"samples": 467,
|
| 1576 |
+
"teacher_exact": 0.24411134903640258,
|
| 1577 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1578 |
+
"final_exact": 0.5867237687366167,
|
| 1579 |
+
"gain_pp": 34.261241970021416,
|
| 1580 |
+
"rewrites": 278,
|
| 1581 |
+
"abstained": 189,
|
| 1582 |
+
"beneficial": 162,
|
| 1583 |
+
"harmful": 2,
|
| 1584 |
+
"rewrite_precision": 0.9878048780487805
|
| 1585 |
+
},
|
| 1586 |
+
{
|
| 1587 |
+
"threshold": 0.3,
|
| 1588 |
+
"samples": 467,
|
| 1589 |
+
"teacher_exact": 0.24411134903640258,
|
| 1590 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1591 |
+
"final_exact": 0.5867237687366167,
|
| 1592 |
+
"gain_pp": 34.261241970021416,
|
| 1593 |
+
"rewrites": 278,
|
| 1594 |
+
"abstained": 189,
|
| 1595 |
+
"beneficial": 162,
|
| 1596 |
+
"harmful": 2,
|
| 1597 |
+
"rewrite_precision": 0.9878048780487805
|
| 1598 |
+
},
|
| 1599 |
+
{
|
| 1600 |
+
"threshold": 0.32,
|
| 1601 |
+
"samples": 467,
|
| 1602 |
+
"teacher_exact": 0.24411134903640258,
|
| 1603 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1604 |
+
"final_exact": 0.5867237687366167,
|
| 1605 |
+
"gain_pp": 34.261241970021416,
|
| 1606 |
+
"rewrites": 278,
|
| 1607 |
+
"abstained": 189,
|
| 1608 |
+
"beneficial": 162,
|
| 1609 |
+
"harmful": 2,
|
| 1610 |
+
"rewrite_precision": 0.9878048780487805
|
| 1611 |
+
},
|
| 1612 |
+
{
|
| 1613 |
+
"threshold": 0.34,
|
| 1614 |
+
"samples": 467,
|
| 1615 |
+
"teacher_exact": 0.24411134903640258,
|
| 1616 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1617 |
+
"final_exact": 0.5867237687366167,
|
| 1618 |
+
"gain_pp": 34.261241970021416,
|
| 1619 |
+
"rewrites": 278,
|
| 1620 |
+
"abstained": 189,
|
| 1621 |
+
"beneficial": 162,
|
| 1622 |
+
"harmful": 2,
|
| 1623 |
+
"rewrite_precision": 0.9878048780487805
|
| 1624 |
+
},
|
| 1625 |
+
{
|
| 1626 |
+
"threshold": 0.36,
|
| 1627 |
+
"samples": 467,
|
| 1628 |
+
"teacher_exact": 0.24411134903640258,
|
| 1629 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1630 |
+
"final_exact": 0.5867237687366167,
|
| 1631 |
+
"gain_pp": 34.261241970021416,
|
| 1632 |
+
"rewrites": 278,
|
| 1633 |
+
"abstained": 189,
|
| 1634 |
+
"beneficial": 162,
|
| 1635 |
+
"harmful": 2,
|
| 1636 |
+
"rewrite_precision": 0.9878048780487805
|
| 1637 |
+
},
|
| 1638 |
+
{
|
| 1639 |
+
"threshold": 0.38,
|
| 1640 |
+
"samples": 467,
|
| 1641 |
+
"teacher_exact": 0.24411134903640258,
|
| 1642 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1643 |
+
"final_exact": 0.5867237687366167,
|
| 1644 |
+
"gain_pp": 34.261241970021416,
|
| 1645 |
+
"rewrites": 278,
|
| 1646 |
+
"abstained": 189,
|
| 1647 |
+
"beneficial": 162,
|
| 1648 |
+
"harmful": 2,
|
| 1649 |
+
"rewrite_precision": 0.9878048780487805
|
| 1650 |
+
},
|
| 1651 |
+
{
|
| 1652 |
+
"threshold": 0.4,
|
| 1653 |
+
"samples": 467,
|
| 1654 |
+
"teacher_exact": 0.24411134903640258,
|
| 1655 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1656 |
+
"final_exact": 0.5867237687366167,
|
| 1657 |
+
"gain_pp": 34.261241970021416,
|
| 1658 |
+
"rewrites": 278,
|
| 1659 |
+
"abstained": 189,
|
| 1660 |
+
"beneficial": 162,
|
| 1661 |
+
"harmful": 2,
|
| 1662 |
+
"rewrite_precision": 0.9878048780487805
|
| 1663 |
+
},
|
| 1664 |
+
{
|
| 1665 |
+
"threshold": 0.42,
|
| 1666 |
+
"samples": 467,
|
| 1667 |
+
"teacher_exact": 0.24411134903640258,
|
| 1668 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1669 |
+
"final_exact": 0.5867237687366167,
|
| 1670 |
+
"gain_pp": 34.261241970021416,
|
| 1671 |
+
"rewrites": 278,
|
| 1672 |
+
"abstained": 189,
|
| 1673 |
+
"beneficial": 162,
|
| 1674 |
+
"harmful": 2,
|
| 1675 |
+
"rewrite_precision": 0.9878048780487805
|
| 1676 |
+
},
|
| 1677 |
+
{
|
| 1678 |
+
"threshold": 0.44,
|
| 1679 |
+
"samples": 467,
|
| 1680 |
+
"teacher_exact": 0.24411134903640258,
|
| 1681 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1682 |
+
"final_exact": 0.5867237687366167,
|
| 1683 |
+
"gain_pp": 34.261241970021416,
|
| 1684 |
+
"rewrites": 278,
|
| 1685 |
+
"abstained": 189,
|
| 1686 |
+
"beneficial": 162,
|
| 1687 |
+
"harmful": 2,
|
| 1688 |
+
"rewrite_precision": 0.9878048780487805
|
| 1689 |
+
},
|
| 1690 |
+
{
|
| 1691 |
+
"threshold": 0.46,
|
| 1692 |
+
"samples": 467,
|
| 1693 |
+
"teacher_exact": 0.24411134903640258,
|
| 1694 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1695 |
+
"final_exact": 0.5867237687366167,
|
| 1696 |
+
"gain_pp": 34.261241970021416,
|
| 1697 |
+
"rewrites": 278,
|
| 1698 |
+
"abstained": 189,
|
| 1699 |
+
"beneficial": 162,
|
| 1700 |
+
"harmful": 2,
|
| 1701 |
+
"rewrite_precision": 0.9878048780487805
|
| 1702 |
+
},
|
| 1703 |
+
{
|
| 1704 |
+
"threshold": 0.48,
|
| 1705 |
+
"samples": 467,
|
| 1706 |
+
"teacher_exact": 0.24411134903640258,
|
| 1707 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1708 |
+
"final_exact": 0.5867237687366167,
|
| 1709 |
+
"gain_pp": 34.261241970021416,
|
| 1710 |
+
"rewrites": 278,
|
| 1711 |
+
"abstained": 189,
|
| 1712 |
+
"beneficial": 162,
|
| 1713 |
+
"harmful": 2,
|
| 1714 |
+
"rewrite_precision": 0.9878048780487805
|
| 1715 |
+
},
|
| 1716 |
+
{
|
| 1717 |
+
"threshold": 0.5,
|
| 1718 |
+
"samples": 467,
|
| 1719 |
+
"teacher_exact": 0.24411134903640258,
|
| 1720 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1721 |
+
"final_exact": 0.5845824411134903,
|
| 1722 |
+
"gain_pp": 34.04710920770878,
|
| 1723 |
+
"rewrites": 277,
|
| 1724 |
+
"abstained": 190,
|
| 1725 |
+
"beneficial": 161,
|
| 1726 |
+
"harmful": 2,
|
| 1727 |
+
"rewrite_precision": 0.9877300613496932
|
| 1728 |
+
},
|
| 1729 |
+
{
|
| 1730 |
+
"threshold": 0.52,
|
| 1731 |
+
"samples": 467,
|
| 1732 |
+
"teacher_exact": 0.24411134903640258,
|
| 1733 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1734 |
+
"final_exact": 0.5845824411134903,
|
| 1735 |
+
"gain_pp": 34.04710920770878,
|
| 1736 |
+
"rewrites": 277,
|
| 1737 |
+
"abstained": 190,
|
| 1738 |
+
"beneficial": 161,
|
| 1739 |
+
"harmful": 2,
|
| 1740 |
+
"rewrite_precision": 0.9877300613496932
|
| 1741 |
+
},
|
| 1742 |
+
{
|
| 1743 |
+
"threshold": 0.54,
|
| 1744 |
+
"samples": 467,
|
| 1745 |
+
"teacher_exact": 0.24411134903640258,
|
| 1746 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1747 |
+
"final_exact": 0.5845824411134903,
|
| 1748 |
+
"gain_pp": 34.04710920770878,
|
| 1749 |
+
"rewrites": 276,
|
| 1750 |
+
"abstained": 191,
|
| 1751 |
+
"beneficial": 161,
|
| 1752 |
+
"harmful": 2,
|
| 1753 |
+
"rewrite_precision": 0.9877300613496932
|
| 1754 |
+
},
|
| 1755 |
+
{
|
| 1756 |
+
"threshold": 0.56,
|
| 1757 |
+
"samples": 467,
|
| 1758 |
+
"teacher_exact": 0.24411134903640258,
|
| 1759 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1760 |
+
"final_exact": 0.5845824411134903,
|
| 1761 |
+
"gain_pp": 34.04710920770878,
|
| 1762 |
+
"rewrites": 274,
|
| 1763 |
+
"abstained": 193,
|
| 1764 |
+
"beneficial": 161,
|
| 1765 |
+
"harmful": 2,
|
| 1766 |
+
"rewrite_precision": 0.9877300613496932
|
| 1767 |
+
},
|
| 1768 |
+
{
|
| 1769 |
+
"threshold": 0.58,
|
| 1770 |
+
"samples": 467,
|
| 1771 |
+
"teacher_exact": 0.24411134903640258,
|
| 1772 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1773 |
+
"final_exact": 0.5845824411134903,
|
| 1774 |
+
"gain_pp": 34.04710920770878,
|
| 1775 |
+
"rewrites": 274,
|
| 1776 |
+
"abstained": 193,
|
| 1777 |
+
"beneficial": 161,
|
| 1778 |
+
"harmful": 2,
|
| 1779 |
+
"rewrite_precision": 0.9877300613496932
|
| 1780 |
+
},
|
| 1781 |
+
{
|
| 1782 |
+
"threshold": 0.6,
|
| 1783 |
+
"samples": 467,
|
| 1784 |
+
"teacher_exact": 0.24411134903640258,
|
| 1785 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1786 |
+
"final_exact": 0.5867237687366167,
|
| 1787 |
+
"gain_pp": 34.261241970021416,
|
| 1788 |
+
"rewrites": 273,
|
| 1789 |
+
"abstained": 194,
|
| 1790 |
+
"beneficial": 161,
|
| 1791 |
+
"harmful": 1,
|
| 1792 |
+
"rewrite_precision": 0.9938271604938271
|
| 1793 |
+
},
|
| 1794 |
+
{
|
| 1795 |
+
"threshold": 0.62,
|
| 1796 |
+
"samples": 467,
|
| 1797 |
+
"teacher_exact": 0.24411134903640258,
|
| 1798 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1799 |
+
"final_exact": 0.5867237687366167,
|
| 1800 |
+
"gain_pp": 34.261241970021416,
|
| 1801 |
+
"rewrites": 273,
|
| 1802 |
+
"abstained": 194,
|
| 1803 |
+
"beneficial": 161,
|
| 1804 |
+
"harmful": 1,
|
| 1805 |
+
"rewrite_precision": 0.9938271604938271
|
| 1806 |
+
},
|
| 1807 |
+
{
|
| 1808 |
+
"threshold": 0.64,
|
| 1809 |
+
"samples": 467,
|
| 1810 |
+
"teacher_exact": 0.24411134903640258,
|
| 1811 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1812 |
+
"final_exact": 0.5867237687366167,
|
| 1813 |
+
"gain_pp": 34.261241970021416,
|
| 1814 |
+
"rewrites": 273,
|
| 1815 |
+
"abstained": 194,
|
| 1816 |
+
"beneficial": 161,
|
| 1817 |
+
"harmful": 1,
|
| 1818 |
+
"rewrite_precision": 0.9938271604938271
|
| 1819 |
+
},
|
| 1820 |
+
{
|
| 1821 |
+
"threshold": 0.66,
|
| 1822 |
+
"samples": 467,
|
| 1823 |
+
"teacher_exact": 0.24411134903640258,
|
| 1824 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1825 |
+
"final_exact": 0.5867237687366167,
|
| 1826 |
+
"gain_pp": 34.261241970021416,
|
| 1827 |
+
"rewrites": 273,
|
| 1828 |
+
"abstained": 194,
|
| 1829 |
+
"beneficial": 161,
|
| 1830 |
+
"harmful": 1,
|
| 1831 |
+
"rewrite_precision": 0.9938271604938271
|
| 1832 |
+
},
|
| 1833 |
+
{
|
| 1834 |
+
"threshold": 0.68,
|
| 1835 |
+
"samples": 467,
|
| 1836 |
+
"teacher_exact": 0.24411134903640258,
|
| 1837 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1838 |
+
"final_exact": 0.5867237687366167,
|
| 1839 |
+
"gain_pp": 34.261241970021416,
|
| 1840 |
+
"rewrites": 272,
|
| 1841 |
+
"abstained": 195,
|
| 1842 |
+
"beneficial": 161,
|
| 1843 |
+
"harmful": 1,
|
| 1844 |
+
"rewrite_precision": 0.9938271604938271
|
| 1845 |
+
},
|
| 1846 |
+
{
|
| 1847 |
+
"threshold": 0.7,
|
| 1848 |
+
"samples": 467,
|
| 1849 |
+
"teacher_exact": 0.24411134903640258,
|
| 1850 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1851 |
+
"final_exact": 0.5867237687366167,
|
| 1852 |
+
"gain_pp": 34.261241970021416,
|
| 1853 |
+
"rewrites": 272,
|
| 1854 |
+
"abstained": 195,
|
| 1855 |
+
"beneficial": 161,
|
| 1856 |
+
"harmful": 1,
|
| 1857 |
+
"rewrite_precision": 0.9938271604938271
|
| 1858 |
+
},
|
| 1859 |
+
{
|
| 1860 |
+
"threshold": 0.72,
|
| 1861 |
+
"samples": 467,
|
| 1862 |
+
"teacher_exact": 0.24411134903640258,
|
| 1863 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1864 |
+
"final_exact": 0.5867237687366167,
|
| 1865 |
+
"gain_pp": 34.261241970021416,
|
| 1866 |
+
"rewrites": 272,
|
| 1867 |
+
"abstained": 195,
|
| 1868 |
+
"beneficial": 161,
|
| 1869 |
+
"harmful": 1,
|
| 1870 |
+
"rewrite_precision": 0.9938271604938271
|
| 1871 |
+
},
|
| 1872 |
+
{
|
| 1873 |
+
"threshold": 0.74,
|
| 1874 |
+
"samples": 467,
|
| 1875 |
+
"teacher_exact": 0.24411134903640258,
|
| 1876 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1877 |
+
"final_exact": 0.5845824411134903,
|
| 1878 |
+
"gain_pp": 34.04710920770878,
|
| 1879 |
+
"rewrites": 271,
|
| 1880 |
+
"abstained": 196,
|
| 1881 |
+
"beneficial": 160,
|
| 1882 |
+
"harmful": 1,
|
| 1883 |
+
"rewrite_precision": 0.9937888198757764
|
| 1884 |
+
},
|
| 1885 |
+
{
|
| 1886 |
+
"threshold": 0.76,
|
| 1887 |
+
"samples": 467,
|
| 1888 |
+
"teacher_exact": 0.24411134903640258,
|
| 1889 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1890 |
+
"final_exact": 0.5845824411134903,
|
| 1891 |
+
"gain_pp": 34.04710920770878,
|
| 1892 |
+
"rewrites": 270,
|
| 1893 |
+
"abstained": 197,
|
| 1894 |
+
"beneficial": 160,
|
| 1895 |
+
"harmful": 1,
|
| 1896 |
+
"rewrite_precision": 0.9937888198757764
|
| 1897 |
+
},
|
| 1898 |
+
{
|
| 1899 |
+
"threshold": 0.78,
|
| 1900 |
+
"samples": 467,
|
| 1901 |
+
"teacher_exact": 0.24411134903640258,
|
| 1902 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1903 |
+
"final_exact": 0.582441113490364,
|
| 1904 |
+
"gain_pp": 33.83297644539615,
|
| 1905 |
+
"rewrites": 269,
|
| 1906 |
+
"abstained": 198,
|
| 1907 |
+
"beneficial": 159,
|
| 1908 |
+
"harmful": 1,
|
| 1909 |
+
"rewrite_precision": 0.99375
|
| 1910 |
+
},
|
| 1911 |
+
{
|
| 1912 |
+
"threshold": 0.8,
|
| 1913 |
+
"samples": 467,
|
| 1914 |
+
"teacher_exact": 0.24411134903640258,
|
| 1915 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1916 |
+
"final_exact": 0.5802997858672377,
|
| 1917 |
+
"gain_pp": 33.61884368308351,
|
| 1918 |
+
"rewrites": 268,
|
| 1919 |
+
"abstained": 199,
|
| 1920 |
+
"beneficial": 158,
|
| 1921 |
+
"harmful": 1,
|
| 1922 |
+
"rewrite_precision": 0.9937106918238994
|
| 1923 |
+
},
|
| 1924 |
+
{
|
| 1925 |
+
"threshold": 0.82,
|
| 1926 |
+
"samples": 467,
|
| 1927 |
+
"teacher_exact": 0.24411134903640258,
|
| 1928 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1929 |
+
"final_exact": 0.5802997858672377,
|
| 1930 |
+
"gain_pp": 33.61884368308351,
|
| 1931 |
+
"rewrites": 268,
|
| 1932 |
+
"abstained": 199,
|
| 1933 |
+
"beneficial": 158,
|
| 1934 |
+
"harmful": 1,
|
| 1935 |
+
"rewrite_precision": 0.9937106918238994
|
| 1936 |
+
},
|
| 1937 |
+
{
|
| 1938 |
+
"threshold": 0.84,
|
| 1939 |
+
"samples": 467,
|
| 1940 |
+
"teacher_exact": 0.24411134903640258,
|
| 1941 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1942 |
+
"final_exact": 0.5802997858672377,
|
| 1943 |
+
"gain_pp": 33.61884368308351,
|
| 1944 |
+
"rewrites": 268,
|
| 1945 |
+
"abstained": 199,
|
| 1946 |
+
"beneficial": 158,
|
| 1947 |
+
"harmful": 1,
|
| 1948 |
+
"rewrite_precision": 0.9937106918238994
|
| 1949 |
+
},
|
| 1950 |
+
{
|
| 1951 |
+
"threshold": 0.86,
|
| 1952 |
+
"samples": 467,
|
| 1953 |
+
"teacher_exact": 0.24411134903640258,
|
| 1954 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1955 |
+
"final_exact": 0.5802997858672377,
|
| 1956 |
+
"gain_pp": 33.61884368308351,
|
| 1957 |
+
"rewrites": 267,
|
| 1958 |
+
"abstained": 200,
|
| 1959 |
+
"beneficial": 158,
|
| 1960 |
+
"harmful": 1,
|
| 1961 |
+
"rewrite_precision": 0.9937106918238994
|
| 1962 |
+
},
|
| 1963 |
+
{
|
| 1964 |
+
"threshold": 0.88,
|
| 1965 |
+
"samples": 467,
|
| 1966 |
+
"teacher_exact": 0.24411134903640258,
|
| 1967 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1968 |
+
"final_exact": 0.576017130620985,
|
| 1969 |
+
"gain_pp": 33.190578158458244,
|
| 1970 |
+
"rewrites": 263,
|
| 1971 |
+
"abstained": 204,
|
| 1972 |
+
"beneficial": 155,
|
| 1973 |
+
"harmful": 0,
|
| 1974 |
+
"rewrite_precision": 1.0
|
| 1975 |
+
},
|
| 1976 |
+
{
|
| 1977 |
+
"threshold": 0.9,
|
| 1978 |
+
"samples": 467,
|
| 1979 |
+
"teacher_exact": 0.24411134903640258,
|
| 1980 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1981 |
+
"final_exact": 0.5717344753747323,
|
| 1982 |
+
"gain_pp": 32.76231263383298,
|
| 1983 |
+
"rewrites": 261,
|
| 1984 |
+
"abstained": 206,
|
| 1985 |
+
"beneficial": 153,
|
| 1986 |
+
"harmful": 0,
|
| 1987 |
+
"rewrite_precision": 1.0
|
| 1988 |
+
},
|
| 1989 |
+
{
|
| 1990 |
+
"threshold": 0.92,
|
| 1991 |
+
"samples": 467,
|
| 1992 |
+
"teacher_exact": 0.24411134903640258,
|
| 1993 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 1994 |
+
"final_exact": 0.5653104925053534,
|
| 1995 |
+
"gain_pp": 32.11991434689507,
|
| 1996 |
+
"rewrites": 256,
|
| 1997 |
+
"abstained": 211,
|
| 1998 |
+
"beneficial": 150,
|
| 1999 |
+
"harmful": 0,
|
| 2000 |
+
"rewrite_precision": 1.0
|
| 2001 |
+
},
|
| 2002 |
+
{
|
| 2003 |
+
"threshold": 0.94,
|
| 2004 |
+
"samples": 467,
|
| 2005 |
+
"teacher_exact": 0.24411134903640258,
|
| 2006 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 2007 |
+
"final_exact": 0.5610278372591007,
|
| 2008 |
+
"gain_pp": 31.69164882226981,
|
| 2009 |
+
"rewrites": 253,
|
| 2010 |
+
"abstained": 214,
|
| 2011 |
+
"beneficial": 148,
|
| 2012 |
+
"harmful": 0,
|
| 2013 |
+
"rewrite_precision": 1.0
|
| 2014 |
+
},
|
| 2015 |
+
{
|
| 2016 |
+
"threshold": 0.96,
|
| 2017 |
+
"samples": 467,
|
| 2018 |
+
"teacher_exact": 0.24411134903640258,
|
| 2019 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 2020 |
+
"final_exact": 0.5546038543897216,
|
| 2021 |
+
"gain_pp": 31.049250535331907,
|
| 2022 |
+
"rewrites": 250,
|
| 2023 |
+
"abstained": 217,
|
| 2024 |
+
"beneficial": 145,
|
| 2025 |
+
"harmful": 0,
|
| 2026 |
+
"rewrite_precision": 1.0
|
| 2027 |
+
},
|
| 2028 |
+
{
|
| 2029 |
+
"threshold": 0.98,
|
| 2030 |
+
"samples": 467,
|
| 2031 |
+
"teacher_exact": 0.24411134903640258,
|
| 2032 |
+
"teacher_visual_family": 0.5952890792291221,
|
| 2033 |
+
"final_exact": 0.5417558886509636,
|
| 2034 |
+
"gain_pp": 29.764453961456102,
|
| 2035 |
+
"rewrites": 238,
|
| 2036 |
+
"abstained": 229,
|
| 2037 |
+
"beneficial": 139,
|
| 2038 |
+
"harmful": 0,
|
| 2039 |
+
"rewrite_precision": 1.0
|
| 2040 |
+
}
|
| 2041 |
+
],
|
| 2042 |
+
"official_test": {
|
| 2043 |
+
"threshold": 0.72,
|
| 2044 |
+
"samples": 635,
|
| 2045 |
+
"teacher_exact": 0.2866141732283465,
|
| 2046 |
+
"teacher_visual_family": 0.5007874015748032,
|
| 2047 |
+
"final_exact": 0.4409448818897638,
|
| 2048 |
+
"gain_pp": 15.433070866141732,
|
| 2049 |
+
"rewrites": 307,
|
| 2050 |
+
"abstained": 328,
|
| 2051 |
+
"beneficial": 125,
|
| 2052 |
+
"harmful": 27,
|
| 2053 |
+
"rewrite_precision": 0.8223684210526315
|
| 2054 |
+
}
|
| 2055 |
+
}
|
| 2056 |
+
],
|
| 2057 |
+
"official_test_summary": {
|
| 2058 |
+
"teacher_exact": {
|
| 2059 |
+
"values": [
|
| 2060 |
+
0.4519685039370079,
|
| 2061 |
+
0.27086614173228346,
|
| 2062 |
+
0.2866141732283465
|
| 2063 |
+
],
|
| 2064 |
+
"mean": 0.33648293963254594
|
| 2065 |
+
},
|
| 2066 |
+
"teacher_visual_family": {
|
| 2067 |
+
"values": [
|
| 2068 |
+
0.6425196850393701,
|
| 2069 |
+
0.44724409448818897,
|
| 2070 |
+
0.5007874015748032
|
| 2071 |
+
],
|
| 2072 |
+
"mean": 0.5301837270341206
|
| 2073 |
+
},
|
| 2074 |
+
"final_exact": {
|
| 2075 |
+
"values": [
|
| 2076 |
+
0.584251968503937,
|
| 2077 |
+
0.3937007874015748,
|
| 2078 |
+
0.4409448818897638
|
| 2079 |
+
],
|
| 2080 |
+
"mean": 0.47296587926509187
|
| 2081 |
+
},
|
| 2082 |
+
"gain_pp": {
|
| 2083 |
+
"values": [
|
| 2084 |
+
13.228346456692913,
|
| 2085 |
+
12.283464566929133,
|
| 2086 |
+
15.433070866141732
|
| 2087 |
+
],
|
| 2088 |
+
"mean": 13.648293963254593
|
| 2089 |
+
},
|
| 2090 |
+
"rewrite_precision": {
|
| 2091 |
+
"values": [
|
| 2092 |
+
0.8181818181818182,
|
| 2093 |
+
0.8,
|
| 2094 |
+
0.8223684210526315
|
| 2095 |
+
],
|
| 2096 |
+
"mean": 0.8135167464114832
|
| 2097 |
+
}
|
| 2098 |
+
},
|
| 2099 |
+
"track": "R_noncommercial_only",
|
| 2100 |
+
"product_validation": false,
|
| 2101 |
+
"distillation_allowed": false
|
| 2102 |
+
}
|
reports/formula_adapter_h64_3seed_summary.json
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "R-MATH-INK-06-FORMULA-ADAPTER-3SEED-001",
|
| 3 |
+
"generated_at": "2026-07-23T21:13:06.294998+00:00",
|
| 4 |
+
"reports": [
|
| 5 |
+
"research\\runs\\math_ink_06_formula_adapter_h64_seed17_20260724\\report.json",
|
| 6 |
+
"research\\runs\\math_ink_06_formula_adapter_h64_seed31_20260724\\report.json",
|
| 7 |
+
"research\\runs\\math_ink_06_formula_adapter_h64_seed47_20260724\\report.json"
|
| 8 |
+
],
|
| 9 |
+
"metrics": {
|
| 10 |
+
"validation_exact_top1": {
|
| 11 |
+
"values": [
|
| 12 |
+
0.8290650844573975,
|
| 13 |
+
0.8537488579750061,
|
| 14 |
+
0.8728787302970886
|
| 15 |
+
],
|
| 16 |
+
"mean": 0.8518975575764974,
|
| 17 |
+
"std": 0.017934684744290146,
|
| 18 |
+
"minimum": 0.8290650844573975
|
| 19 |
+
},
|
| 20 |
+
"validation_family_head_top1": {
|
| 21 |
+
"values": [
|
| 22 |
+
0.931502640247345,
|
| 23 |
+
0.9339709877967834,
|
| 24 |
+
0.9348966479301453
|
| 25 |
+
],
|
| 26 |
+
"mean": 0.9334567586580912,
|
| 27 |
+
"std": 0.001432514279867618,
|
| 28 |
+
"minimum": 0.931502640247345
|
| 29 |
+
},
|
| 30 |
+
"validation_visual_family_top1": {
|
| 31 |
+
"values": [
|
| 32 |
+
0.9302684356680037,
|
| 33 |
+
0.937365010799136,
|
| 34 |
+
0.9315026226473311
|
| 35 |
+
],
|
| 36 |
+
"mean": 0.9330453563714902,
|
| 37 |
+
"std": 0.003095735255228302,
|
| 38 |
+
"minimum": 0.9302684356680037
|
| 39 |
+
},
|
| 40 |
+
"official_test_exact_top1": {
|
| 41 |
+
"values": [
|
| 42 |
+
0.8113494515419006,
|
| 43 |
+
0.824860692024231,
|
| 44 |
+
0.8351629972457886
|
| 45 |
+
],
|
| 46 |
+
"mean": 0.8237910469373068,
|
| 47 |
+
"std": 0.009751216854211096,
|
| 48 |
+
"minimum": 0.8113494515419006
|
| 49 |
+
},
|
| 50 |
+
"official_test_family_head_top1": {
|
| 51 |
+
"values": [
|
| 52 |
+
0.880256712436676,
|
| 53 |
+
0.8805944919586182,
|
| 54 |
+
0.8816078305244446
|
| 55 |
+
],
|
| 56 |
+
"mean": 0.8808196783065796,
|
| 57 |
+
"std": 0.0005741147912250024,
|
| 58 |
+
"minimum": 0.880256712436676
|
| 59 |
+
},
|
| 60 |
+
"official_test_visual_family_top1": {
|
| 61 |
+
"values": [
|
| 62 |
+
0.8809322749535552,
|
| 63 |
+
0.8839723019760176,
|
| 64 |
+
0.8802567133930079
|
| 65 |
+
],
|
| 66 |
+
"mean": 0.8817204301075269,
|
| 67 |
+
"std": 0.0016160220789739894,
|
| 68 |
+
"minimum": 0.8802567133930079
|
| 69 |
+
},
|
| 70 |
+
"visual_family_generalization_gap_pp": {
|
| 71 |
+
"values": [
|
| 72 |
+
4.933616071444846,
|
| 73 |
+
5.339270882311842,
|
| 74 |
+
5.124590925432315
|
| 75 |
+
],
|
| 76 |
+
"mean": 5.132492626396335,
|
| 77 |
+
"std": 0.16570211033353024,
|
| 78 |
+
"maximum": 5.339270882311842
|
| 79 |
+
}
|
| 80 |
+
},
|
| 81 |
+
"decision": {
|
| 82 |
+
"all_seed_validation_92_passed": true,
|
| 83 |
+
"all_seed_official_test_92_passed": false,
|
| 84 |
+
"architecture_recoverable": true,
|
| 85 |
+
"writer_domain_generalization_gate_passed": false,
|
| 86 |
+
"product_validation": false,
|
| 87 |
+
"distillation_allowed": false,
|
| 88 |
+
"next_gate": "P 연속식 writer/device-disjoint formula adapter 학습·평가"
|
| 89 |
+
},
|
| 90 |
+
"track": "R_noncommercial_only",
|
| 91 |
+
"product_validation": false
|
| 92 |
+
}
|
reports/formula_adapter_h64_seed17.json
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "R-MATH-INK-06-FORMULA-ADAPTER-001",
|
| 3 |
+
"generated_at": "2026-07-23T21:07:54.418028+00:00",
|
| 4 |
+
"seed": 17,
|
| 5 |
+
"device": "cuda",
|
| 6 |
+
"cuda_device": "NVIDIA GeForce GTX 1650",
|
| 7 |
+
"split_contract": "CROHME trainData writer fit/validation; testDataGT one-shot",
|
| 8 |
+
"samples": {
|
| 9 |
+
"fit": 12526,
|
| 10 |
+
"validation": 3241,
|
| 11 |
+
"official_test": 5921
|
| 12 |
+
},
|
| 13 |
+
"label_support": {
|
| 14 |
+
"fit": 69,
|
| 15 |
+
"validation": 64,
|
| 16 |
+
"official_test": 69
|
| 17 |
+
},
|
| 18 |
+
"baseline_validation": {
|
| 19 |
+
"samples": 3241,
|
| 20 |
+
"exact_top1": 0.5489046573638916,
|
| 21 |
+
"family_head_top1": 0.607528567314148,
|
| 22 |
+
"visual_family_top1": 0.6291268127121259
|
| 23 |
+
},
|
| 24 |
+
"selected_epoch": 14,
|
| 25 |
+
"selected_validation": {
|
| 26 |
+
"samples": 3241,
|
| 27 |
+
"exact_top1": 0.8290650844573975,
|
| 28 |
+
"family_head_top1": 0.931502640247345,
|
| 29 |
+
"visual_family_top1": 0.9302684356680037
|
| 30 |
+
},
|
| 31 |
+
"official_test": {
|
| 32 |
+
"samples": 5921,
|
| 33 |
+
"exact_top1": 0.8113494515419006,
|
| 34 |
+
"family_head_top1": 0.880256712436676,
|
| 35 |
+
"visual_family_top1": 0.8809322749535552
|
| 36 |
+
},
|
| 37 |
+
"official_test_skipped": false,
|
| 38 |
+
"history": [
|
| 39 |
+
{
|
| 40 |
+
"epoch": 1,
|
| 41 |
+
"loss": 1.4087682196072169,
|
| 42 |
+
"validation": {
|
| 43 |
+
"samples": 3241,
|
| 44 |
+
"exact_top1": 0.6960814595222473,
|
| 45 |
+
"family_head_top1": 0.8003702759742737,
|
| 46 |
+
"visual_family_top1": 0.8117864856525764
|
| 47 |
+
}
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"epoch": 2,
|
| 51 |
+
"loss": 0.9927104310113557,
|
| 52 |
+
"validation": {
|
| 53 |
+
"samples": 3241,
|
| 54 |
+
"exact_top1": 0.7294045090675354,
|
| 55 |
+
"family_head_top1": 0.8435667753219604,
|
| 56 |
+
"visual_family_top1": 0.8636223387843258
|
| 57 |
+
}
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"epoch": 3,
|
| 61 |
+
"loss": 0.870262260339698,
|
| 62 |
+
"validation": {
|
| 63 |
+
"samples": 3241,
|
| 64 |
+
"exact_top1": 0.7488429546356201,
|
| 65 |
+
"family_head_top1": 0.8704103827476501,
|
| 66 |
+
"visual_family_top1": 0.8756556618327677
|
| 67 |
+
}
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"epoch": 4,
|
| 71 |
+
"loss": 0.7446754088207167,
|
| 72 |
+
"validation": {
|
| 73 |
+
"samples": 3241,
|
| 74 |
+
"exact_top1": 0.773526668548584,
|
| 75 |
+
"family_head_top1": 0.8842949867248535,
|
| 76 |
+
"visual_family_top1": 0.8944770132675101
|
| 77 |
+
}
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"epoch": 5,
|
| 81 |
+
"loss": 0.6735653883340408,
|
| 82 |
+
"validation": {
|
| 83 |
+
"samples": 3241,
|
| 84 |
+
"exact_top1": 0.7784634232521057,
|
| 85 |
+
"family_head_top1": 0.9003394246101379,
|
| 86 |
+
"visual_family_top1": 0.908053070040111
|
| 87 |
+
}
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"epoch": 6,
|
| 91 |
+
"loss": 0.5923128091559118,
|
| 92 |
+
"validation": {
|
| 93 |
+
"samples": 3241,
|
| 94 |
+
"exact_top1": 0.791422426700592,
|
| 95 |
+
"family_head_top1": 0.9065103530883789,
|
| 96 |
+
"visual_family_top1": 0.9139154581919161
|
| 97 |
+
}
|
| 98 |
+
},
|
| 99 |
+
{
|
| 100 |
+
"epoch": 7,
|
| 101 |
+
"loss": 0.5416236027162902,
|
| 102 |
+
"validation": {
|
| 103 |
+
"samples": 3241,
|
| 104 |
+
"exact_top1": 0.795433521270752,
|
| 105 |
+
"family_head_top1": 0.9129897952079773,
|
| 106 |
+
"visual_family_top1": 0.910212897253934
|
| 107 |
+
}
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"epoch": 8,
|
| 111 |
+
"loss": 0.5023860353596357,
|
| 112 |
+
"validation": {
|
| 113 |
+
"samples": 3241,
|
| 114 |
+
"exact_top1": 0.8093181252479553,
|
| 115 |
+
"family_head_top1": 0.9182351231575012,
|
| 116 |
+
"visual_family_top1": 0.9194692995988892
|
| 117 |
+
}
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"epoch": 9,
|
| 121 |
+
"loss": 0.4808114529872427,
|
| 122 |
+
"validation": {
|
| 123 |
+
"samples": 3241,
|
| 124 |
+
"exact_top1": 0.8127121329307556,
|
| 125 |
+
"family_head_top1": 0.917926549911499,
|
| 126 |
+
"visual_family_top1": 0.9216291268127121
|
| 127 |
+
}
|
| 128 |
+
},
|
| 129 |
+
{
|
| 130 |
+
"epoch": 10,
|
| 131 |
+
"loss": 0.45815988706082716,
|
| 132 |
+
"validation": {
|
| 133 |
+
"samples": 3241,
|
| 134 |
+
"exact_top1": 0.8173403143882751,
|
| 135 |
+
"family_head_top1": 0.9244060516357422,
|
| 136 |
+
"visual_family_top1": 0.9213205800678803
|
| 137 |
+
}
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"epoch": 11,
|
| 141 |
+
"loss": 0.434234369774254,
|
| 142 |
+
"validation": {
|
| 143 |
+
"samples": 3241,
|
| 144 |
+
"exact_top1": 0.8213514089584351,
|
| 145 |
+
"family_head_top1": 0.9265658855438232,
|
| 146 |
+
"visual_family_top1": 0.9244060475161987
|
| 147 |
+
}
|
| 148 |
+
},
|
| 149 |
+
{
|
| 150 |
+
"epoch": 12,
|
| 151 |
+
"loss": 0.41754808657023373,
|
| 152 |
+
"validation": {
|
| 153 |
+
"samples": 3241,
|
| 154 |
+
"exact_top1": 0.8244369029998779,
|
| 155 |
+
"family_head_top1": 0.9268743991851807,
|
| 156 |
+
"visual_family_top1": 0.923788954026535
|
| 157 |
+
}
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"epoch": 13,
|
| 161 |
+
"loss": 0.40670519945572836,
|
| 162 |
+
"validation": {
|
| 163 |
+
"samples": 3241,
|
| 164 |
+
"exact_top1": 0.8241283297538757,
|
| 165 |
+
"family_head_top1": 0.92409747838974,
|
| 166 |
+
"visual_family_top1": 0.9222462203023758
|
| 167 |
+
}
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"epoch": 14,
|
| 171 |
+
"loss": 0.3943827273894329,
|
| 172 |
+
"validation": {
|
| 173 |
+
"samples": 3241,
|
| 174 |
+
"exact_top1": 0.8290650844573975,
|
| 175 |
+
"family_head_top1": 0.931502640247345,
|
| 176 |
+
"visual_family_top1": 0.9302684356680037
|
| 177 |
+
}
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"epoch": 15,
|
| 181 |
+
"loss": 0.38740119520498784,
|
| 182 |
+
"validation": {
|
| 183 |
+
"samples": 3241,
|
| 184 |
+
"exact_top1": 0.8278309106826782,
|
| 185 |
+
"family_head_top1": 0.9296513199806213,
|
| 186 |
+
"visual_family_top1": 0.9302684356680037
|
| 187 |
+
}
|
| 188 |
+
},
|
| 189 |
+
{
|
| 190 |
+
"epoch": 16,
|
| 191 |
+
"loss": 0.37666105190101934,
|
| 192 |
+
"validation": {
|
| 193 |
+
"samples": 3241,
|
| 194 |
+
"exact_top1": 0.8302993178367615,
|
| 195 |
+
"family_head_top1": 0.9278000593185425,
|
| 196 |
+
"visual_family_top1": 0.9274915149645171
|
| 197 |
+
}
|
| 198 |
+
}
|
| 199 |
+
],
|
| 200 |
+
"checkpoint": "formula_adapter.pt",
|
| 201 |
+
"checkpoint_bytes": 92162,
|
| 202 |
+
"track": "R_noncommercial_only",
|
| 203 |
+
"product_validation": false,
|
| 204 |
+
"distillation_allowed": false
|
| 205 |
+
}
|
reports/formula_adapter_h64_seed31.json
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "R-MATH-INK-06-FORMULA-ADAPTER-001",
|
| 3 |
+
"generated_at": "2026-07-23T21:10:09.642194+00:00",
|
| 4 |
+
"seed": 31,
|
| 5 |
+
"device": "cuda",
|
| 6 |
+
"cuda_device": "NVIDIA GeForce GTX 1650",
|
| 7 |
+
"split_contract": "CROHME trainData writer fit/validation; testDataGT one-shot",
|
| 8 |
+
"samples": {
|
| 9 |
+
"fit": 12526,
|
| 10 |
+
"validation": 3241,
|
| 11 |
+
"official_test": 5921
|
| 12 |
+
},
|
| 13 |
+
"label_support": {
|
| 14 |
+
"fit": 69,
|
| 15 |
+
"validation": 64,
|
| 16 |
+
"official_test": 69
|
| 17 |
+
},
|
| 18 |
+
"baseline_validation": {
|
| 19 |
+
"samples": 3241,
|
| 20 |
+
"exact_top1": 0.534711480140686,
|
| 21 |
+
"family_head_top1": 0.5908670425415039,
|
| 22 |
+
"visual_family_top1": 0.6143165689601975
|
| 23 |
+
},
|
| 24 |
+
"selected_epoch": 16,
|
| 25 |
+
"selected_validation": {
|
| 26 |
+
"samples": 3241,
|
| 27 |
+
"exact_top1": 0.8537488579750061,
|
| 28 |
+
"family_head_top1": 0.9339709877967834,
|
| 29 |
+
"visual_family_top1": 0.937365010799136
|
| 30 |
+
},
|
| 31 |
+
"official_test": {
|
| 32 |
+
"samples": 5921,
|
| 33 |
+
"exact_top1": 0.824860692024231,
|
| 34 |
+
"family_head_top1": 0.8805944919586182,
|
| 35 |
+
"visual_family_top1": 0.8839723019760176
|
| 36 |
+
},
|
| 37 |
+
"official_test_skipped": false,
|
| 38 |
+
"history": [
|
| 39 |
+
{
|
| 40 |
+
"epoch": 1,
|
| 41 |
+
"loss": 1.315348552197826,
|
| 42 |
+
"validation": {
|
| 43 |
+
"samples": 3241,
|
| 44 |
+
"exact_top1": 0.6806541085243225,
|
| 45 |
+
"family_head_top1": 0.7877198457717896,
|
| 46 |
+
"visual_family_top1": 0.81240357914224
|
| 47 |
+
}
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"epoch": 2,
|
| 51 |
+
"loss": 0.9780515262058803,
|
| 52 |
+
"validation": {
|
| 53 |
+
"samples": 3241,
|
| 54 |
+
"exact_top1": 0.7309472560882568,
|
| 55 |
+
"family_head_top1": 0.8463436961174011,
|
| 56 |
+
"visual_family_top1": 0.8731872878741129
|
| 57 |
+
}
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"epoch": 3,
|
| 61 |
+
"loss": 0.8133625230010675,
|
| 62 |
+
"validation": {
|
| 63 |
+
"samples": 3241,
|
| 64 |
+
"exact_top1": 0.7540882229804993,
|
| 65 |
+
"family_head_top1": 0.8750385642051697,
|
| 66 |
+
"visual_family_top1": 0.8855291576673866
|
| 67 |
+
}
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"epoch": 4,
|
| 71 |
+
"loss": 0.72881653114241,
|
| 72 |
+
"validation": {
|
| 73 |
+
"samples": 3241,
|
| 74 |
+
"exact_top1": 0.7710583209991455,
|
| 75 |
+
"family_head_top1": 0.8824436664581299,
|
| 76 |
+
"visual_family_top1": 0.9018821351434743
|
| 77 |
+
}
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"epoch": 5,
|
| 81 |
+
"loss": 0.6326125519616264,
|
| 82 |
+
"validation": {
|
| 83 |
+
"samples": 3241,
|
| 84 |
+
"exact_top1": 0.7926566004753113,
|
| 85 |
+
"family_head_top1": 0.9031163454055786,
|
| 86 |
+
"visual_family_top1": 0.9095958037642703
|
| 87 |
+
}
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"epoch": 6,
|
| 91 |
+
"loss": 0.5851733131068093,
|
| 92 |
+
"validation": {
|
| 93 |
+
"samples": 3241,
|
| 94 |
+
"exact_top1": 0.8154890537261963,
|
| 95 |
+
"family_head_top1": 0.9105214476585388,
|
| 96 |
+
"visual_family_top1": 0.9191607528540574
|
| 97 |
+
}
|
| 98 |
+
},
|
| 99 |
+
{
|
| 100 |
+
"epoch": 7,
|
| 101 |
+
"loss": 0.5198692186754577,
|
| 102 |
+
"validation": {
|
| 103 |
+
"samples": 3241,
|
| 104 |
+
"exact_top1": 0.8290650844573975,
|
| 105 |
+
"family_head_top1": 0.9197778701782227,
|
| 106 |
+
"visual_family_top1": 0.9287257019438445
|
| 107 |
+
}
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"epoch": 8,
|
| 111 |
+
"loss": 0.5133049621873972,
|
| 112 |
+
"validation": {
|
| 113 |
+
"samples": 3241,
|
| 114 |
+
"exact_top1": 0.8253625631332397,
|
| 115 |
+
"family_head_top1": 0.9176180362701416,
|
| 116 |
+
"visual_family_top1": 0.9213205800678803
|
| 117 |
+
}
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"epoch": 9,
|
| 121 |
+
"loss": 0.47733115845797014,
|
| 122 |
+
"validation": {
|
| 123 |
+
"samples": 3241,
|
| 124 |
+
"exact_top1": 0.8377044200897217,
|
| 125 |
+
"family_head_top1": 0.9173094630241394,
|
| 126 |
+
"visual_family_top1": 0.925948781240358
|
| 127 |
+
}
|
| 128 |
+
},
|
| 129 |
+
{
|
| 130 |
+
"epoch": 10,
|
| 131 |
+
"loss": 0.44930898535008335,
|
| 132 |
+
"validation": {
|
| 133 |
+
"samples": 3241,
|
| 134 |
+
"exact_top1": 0.8580685257911682,
|
| 135 |
+
"family_head_top1": 0.9278000593185425,
|
| 136 |
+
"visual_family_top1": 0.9352051835853131
|
| 137 |
+
}
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"epoch": 11,
|
| 141 |
+
"loss": 0.4476292309712391,
|
| 142 |
+
"validation": {
|
| 143 |
+
"samples": 3241,
|
| 144 |
+
"exact_top1": 0.8460351824760437,
|
| 145 |
+
"family_head_top1": 0.9256402254104614,
|
| 146 |
+
"visual_family_top1": 0.9324282628818266
|
| 147 |
+
}
|
| 148 |
+
},
|
| 149 |
+
{
|
| 150 |
+
"epoch": 12,
|
| 151 |
+
"loss": 0.42359211432690524,
|
| 152 |
+
"validation": {
|
| 153 |
+
"samples": 3241,
|
| 154 |
+
"exact_top1": 0.8528231978416443,
|
| 155 |
+
"family_head_top1": 0.9278000593185425,
|
| 156 |
+
"visual_family_top1": 0.9358222770749769
|
| 157 |
+
}
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"epoch": 13,
|
| 161 |
+
"loss": 0.4136106882776533,
|
| 162 |
+
"validation": {
|
| 163 |
+
"samples": 3241,
|
| 164 |
+
"exact_top1": 0.8522061109542847,
|
| 165 |
+
"family_head_top1": 0.9287257194519043,
|
| 166 |
+
"visual_family_top1": 0.9342795433508176
|
| 167 |
+
}
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"epoch": 14,
|
| 171 |
+
"loss": 0.4108709236797021,
|
| 172 |
+
"validation": {
|
| 173 |
+
"samples": 3241,
|
| 174 |
+
"exact_top1": 0.8534402847290039,
|
| 175 |
+
"family_head_top1": 0.931502640247345,
|
| 176 |
+
"visual_family_top1": 0.9382906510336316
|
| 177 |
+
}
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"epoch": 15,
|
| 181 |
+
"loss": 0.38455477478552835,
|
| 182 |
+
"validation": {
|
| 183 |
+
"samples": 3241,
|
| 184 |
+
"exact_top1": 0.8518975377082825,
|
| 185 |
+
"family_head_top1": 0.9330453276634216,
|
| 186 |
+
"visual_family_top1": 0.9352051835853131
|
| 187 |
+
}
|
| 188 |
+
},
|
| 189 |
+
{
|
| 190 |
+
"epoch": 16,
|
| 191 |
+
"loss": 0.3846216317342252,
|
| 192 |
+
"validation": {
|
| 193 |
+
"samples": 3241,
|
| 194 |
+
"exact_top1": 0.8537488579750061,
|
| 195 |
+
"family_head_top1": 0.9339709877967834,
|
| 196 |
+
"visual_family_top1": 0.937365010799136
|
| 197 |
+
}
|
| 198 |
+
}
|
| 199 |
+
],
|
| 200 |
+
"checkpoint": "formula_adapter.pt",
|
| 201 |
+
"checkpoint_bytes": 92162,
|
| 202 |
+
"track": "R_noncommercial_only",
|
| 203 |
+
"product_validation": false,
|
| 204 |
+
"distillation_allowed": false
|
| 205 |
+
}
|
reports/formula_adapter_h64_seed47.json
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "R-MATH-INK-06-FORMULA-ADAPTER-001",
|
| 3 |
+
"generated_at": "2026-07-23T21:12:27.630518+00:00",
|
| 4 |
+
"seed": 47,
|
| 5 |
+
"device": "cuda",
|
| 6 |
+
"cuda_device": "NVIDIA GeForce GTX 1650",
|
| 7 |
+
"split_contract": "CROHME trainData writer fit/validation; testDataGT one-shot",
|
| 8 |
+
"samples": {
|
| 9 |
+
"fit": 12526,
|
| 10 |
+
"validation": 3241,
|
| 11 |
+
"official_test": 5921
|
| 12 |
+
},
|
| 13 |
+
"label_support": {
|
| 14 |
+
"fit": 69,
|
| 15 |
+
"validation": 64,
|
| 16 |
+
"official_test": 69
|
| 17 |
+
},
|
| 18 |
+
"baseline_validation": {
|
| 19 |
+
"samples": 3241,
|
| 20 |
+
"exact_top1": 0.5646405220031738,
|
| 21 |
+
"family_head_top1": 0.6093798279762268,
|
| 22 |
+
"visual_family_top1": 0.6430114162295588
|
| 23 |
+
},
|
| 24 |
+
"selected_epoch": 16,
|
| 25 |
+
"selected_validation": {
|
| 26 |
+
"samples": 3241,
|
| 27 |
+
"exact_top1": 0.8728787302970886,
|
| 28 |
+
"family_head_top1": 0.9348966479301453,
|
| 29 |
+
"visual_family_top1": 0.9315026226473311
|
| 30 |
+
},
|
| 31 |
+
"official_test": {
|
| 32 |
+
"samples": 5921,
|
| 33 |
+
"exact_top1": 0.8351629972457886,
|
| 34 |
+
"family_head_top1": 0.8816078305244446,
|
| 35 |
+
"visual_family_top1": 0.8802567133930079
|
| 36 |
+
},
|
| 37 |
+
"official_test_skipped": false,
|
| 38 |
+
"history": [
|
| 39 |
+
{
|
| 40 |
+
"epoch": 1,
|
| 41 |
+
"loss": 1.351943042813515,
|
| 42 |
+
"validation": {
|
| 43 |
+
"samples": 3241,
|
| 44 |
+
"exact_top1": 0.6868250370025635,
|
| 45 |
+
"family_head_top1": 0.7889540195465088,
|
| 46 |
+
"visual_family_top1": 0.8114779389077446
|
| 47 |
+
}
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"epoch": 2,
|
| 51 |
+
"loss": 0.9095250976328947,
|
| 52 |
+
"validation": {
|
| 53 |
+
"samples": 3241,
|
| 54 |
+
"exact_top1": 0.7488429546356201,
|
| 55 |
+
"family_head_top1": 0.8642394542694092,
|
| 56 |
+
"visual_family_top1": 0.8802838630052453
|
| 57 |
+
}
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"epoch": 3,
|
| 61 |
+
"loss": 0.7641888577110914,
|
| 62 |
+
"validation": {
|
| 63 |
+
"samples": 3241,
|
| 64 |
+
"exact_top1": 0.790188193321228,
|
| 65 |
+
"family_head_top1": 0.8947855830192566,
|
| 66 |
+
"visual_family_top1": 0.9058932428262881
|
| 67 |
+
}
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"epoch": 4,
|
| 71 |
+
"loss": 0.6510059152330671,
|
| 72 |
+
"validation": {
|
| 73 |
+
"samples": 3241,
|
| 74 |
+
"exact_top1": 0.7991360425949097,
|
| 75 |
+
"family_head_top1": 0.9068188667297363,
|
| 76 |
+
"visual_family_top1": 0.9163838321505708
|
| 77 |
+
}
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"epoch": 5,
|
| 81 |
+
"loss": 0.5757968784595022,
|
| 82 |
+
"validation": {
|
| 83 |
+
"samples": 3241,
|
| 84 |
+
"exact_top1": 0.8198087215423584,
|
| 85 |
+
"family_head_top1": 0.9120641946792603,
|
| 86 |
+
"visual_family_top1": 0.9203949398333847
|
| 87 |
+
}
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"epoch": 6,
|
| 91 |
+
"loss": 0.5340398185107172,
|
| 92 |
+
"validation": {
|
| 93 |
+
"samples": 3241,
|
| 94 |
+
"exact_top1": 0.8296821713447571,
|
| 95 |
+
"family_head_top1": 0.9210120439529419,
|
| 96 |
+
"visual_family_top1": 0.9207034865782165
|
| 97 |
+
}
|
| 98 |
+
},
|
| 99 |
+
{
|
| 100 |
+
"epoch": 7,
|
| 101 |
+
"loss": 0.48838041144974376,
|
| 102 |
+
"validation": {
|
| 103 |
+
"samples": 3241,
|
| 104 |
+
"exact_top1": 0.8333847522735596,
|
| 105 |
+
"family_head_top1": 0.9176180362701416,
|
| 106 |
+
"visual_family_top1": 0.91792656587473
|
| 107 |
+
}
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"epoch": 8,
|
| 111 |
+
"loss": 0.45159442144997264,
|
| 112 |
+
"validation": {
|
| 113 |
+
"samples": 3241,
|
| 114 |
+
"exact_top1": 0.8528231978416443,
|
| 115 |
+
"family_head_top1": 0.9274914860725403,
|
| 116 |
+
"visual_family_top1": 0.9262573279851898
|
| 117 |
+
}
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"epoch": 9,
|
| 121 |
+
"loss": 0.4143643750219929,
|
| 122 |
+
"validation": {
|
| 123 |
+
"samples": 3241,
|
| 124 |
+
"exact_top1": 0.857759952545166,
|
| 125 |
+
"family_head_top1": 0.926257312297821,
|
| 126 |
+
"visual_family_top1": 0.9240975007713669
|
| 127 |
+
}
|
| 128 |
+
},
|
| 129 |
+
{
|
| 130 |
+
"epoch": 10,
|
| 131 |
+
"loss": 0.3901189827189154,
|
| 132 |
+
"validation": {
|
| 133 |
+
"samples": 3241,
|
| 134 |
+
"exact_top1": 0.8611539602279663,
|
| 135 |
+
"family_head_top1": 0.9293428063392639,
|
| 136 |
+
"visual_family_top1": 0.9293427954335082
|
| 137 |
+
}
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"epoch": 11,
|
| 141 |
+
"loss": 0.403350510767528,
|
| 142 |
+
"validation": {
|
| 143 |
+
"samples": 3241,
|
| 144 |
+
"exact_top1": 0.8617710471153259,
|
| 145 |
+
"family_head_top1": 0.9311940670013428,
|
| 146 |
+
"visual_family_top1": 0.9302684356680037
|
| 147 |
+
}
|
| 148 |
+
},
|
| 149 |
+
{
|
| 150 |
+
"epoch": 12,
|
| 151 |
+
"loss": 0.3805522480789496,
|
| 152 |
+
"validation": {
|
| 153 |
+
"samples": 3241,
|
| 154 |
+
"exact_top1": 0.8667078018188477,
|
| 155 |
+
"family_head_top1": 0.9318111538887024,
|
| 156 |
+
"visual_family_top1": 0.9293427954335082
|
| 157 |
+
}
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"epoch": 13,
|
| 161 |
+
"loss": 0.37446980147945635,
|
| 162 |
+
"validation": {
|
| 163 |
+
"samples": 3241,
|
| 164 |
+
"exact_top1": 0.870101809501648,
|
| 165 |
+
"family_head_top1": 0.9321197271347046,
|
| 166 |
+
"visual_family_top1": 0.9352051835853131
|
| 167 |
+
}
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"epoch": 14,
|
| 171 |
+
"loss": 0.3581130741810312,
|
| 172 |
+
"validation": {
|
| 173 |
+
"samples": 3241,
|
| 174 |
+
"exact_top1": 0.8716445565223694,
|
| 175 |
+
"family_head_top1": 0.933662474155426,
|
| 176 |
+
"visual_family_top1": 0.9324282628818266
|
| 177 |
+
}
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"epoch": 15,
|
| 181 |
+
"loss": 0.34867141137317736,
|
| 182 |
+
"validation": {
|
| 183 |
+
"samples": 3241,
|
| 184 |
+
"exact_top1": 0.872261643409729,
|
| 185 |
+
"family_head_top1": 0.931502640247345,
|
| 186 |
+
"visual_family_top1": 0.9308855291576674
|
| 187 |
+
}
|
| 188 |
+
},
|
| 189 |
+
{
|
| 190 |
+
"epoch": 16,
|
| 191 |
+
"loss": 0.339629308909786,
|
| 192 |
+
"validation": {
|
| 193 |
+
"samples": 3241,
|
| 194 |
+
"exact_top1": 0.8728787302970886,
|
| 195 |
+
"family_head_top1": 0.9348966479301453,
|
| 196 |
+
"visual_family_top1": 0.9315026226473311
|
| 197 |
+
}
|
| 198 |
+
}
|
| 199 |
+
],
|
| 200 |
+
"checkpoint": "formula_adapter.pt",
|
| 201 |
+
"checkpoint_bytes": 92162,
|
| 202 |
+
"track": "R_noncommercial_only",
|
| 203 |
+
"product_validation": false,
|
| 204 |
+
"distillation_allowed": false
|
| 205 |
+
}
|
reports/formula_context_contract_3seed.json
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "R-MATH-INK-06-FORMULA-CONTEXT-CONTRACT-001",
|
| 3 |
+
"generated_at": "2026-07-23T20:58:38.398037+00:00",
|
| 4 |
+
"split_contract": "CROHME trainData writer-validation mode selection; testDataGT one-shot",
|
| 5 |
+
"context_modes": [
|
| 6 |
+
"formula",
|
| 7 |
+
"missing",
|
| 8 |
+
"local_full",
|
| 9 |
+
"height_only"
|
| 10 |
+
],
|
| 11 |
+
"validation_sweep": [
|
| 12 |
+
{
|
| 13 |
+
"mode": "formula",
|
| 14 |
+
"all_supported": {
|
| 15 |
+
"samples": 3241,
|
| 16 |
+
"exact_top1": 0.5637149028077754,
|
| 17 |
+
"visual_family_top1": 0.6457883369330454
|
| 18 |
+
},
|
| 19 |
+
"behavior_targets": {
|
| 20 |
+
"samples": 467,
|
| 21 |
+
"exact_top1": 0.284796573875803,
|
| 22 |
+
"visual_family_top1": 0.6552462526766595
|
| 23 |
+
}
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"mode": "missing",
|
| 27 |
+
"all_supported": {
|
| 28 |
+
"samples": 3241,
|
| 29 |
+
"exact_top1": 0.6078370873187288,
|
| 30 |
+
"visual_family_top1": 0.7655044739278001
|
| 31 |
+
},
|
| 32 |
+
"behavior_targets": {
|
| 33 |
+
"samples": 467,
|
| 34 |
+
"exact_top1": 0.18629550321199143,
|
| 35 |
+
"visual_family_top1": 0.44325481798715205
|
| 36 |
+
}
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"mode": "local_full",
|
| 40 |
+
"all_supported": {
|
| 41 |
+
"samples": 3241,
|
| 42 |
+
"exact_top1": 0.5387226164763962,
|
| 43 |
+
"visual_family_top1": 0.6525763653193459
|
| 44 |
+
},
|
| 45 |
+
"behavior_targets": {
|
| 46 |
+
"samples": 467,
|
| 47 |
+
"exact_top1": 0.08137044967880086,
|
| 48 |
+
"visual_family_top1": 0.5396145610278372
|
| 49 |
+
}
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"mode": "height_only",
|
| 53 |
+
"all_supported": {
|
| 54 |
+
"samples": 3241,
|
| 55 |
+
"exact_top1": 0.5029311940759025,
|
| 56 |
+
"visual_family_top1": 0.5766738660907127
|
| 57 |
+
},
|
| 58 |
+
"behavior_targets": {
|
| 59 |
+
"samples": 467,
|
| 60 |
+
"exact_top1": 0.042826552462526764,
|
| 61 |
+
"visual_family_top1": 0.41541755888650966
|
| 62 |
+
}
|
| 63 |
+
}
|
| 64 |
+
],
|
| 65 |
+
"selected_mode": "formula",
|
| 66 |
+
"selected_validation": {
|
| 67 |
+
"mode": "formula",
|
| 68 |
+
"all_supported": {
|
| 69 |
+
"samples": 3241,
|
| 70 |
+
"exact_top1": 0.5637149028077754,
|
| 71 |
+
"visual_family_top1": 0.6457883369330454
|
| 72 |
+
},
|
| 73 |
+
"behavior_targets": {
|
| 74 |
+
"samples": 467,
|
| 75 |
+
"exact_top1": 0.284796573875803,
|
| 76 |
+
"visual_family_top1": 0.6552462526766595
|
| 77 |
+
}
|
| 78 |
+
},
|
| 79 |
+
"validation_branch_oracle": {
|
| 80 |
+
"all_supported": {
|
| 81 |
+
"samples": 3241,
|
| 82 |
+
"exact_oracle": 0.7682813946312866,
|
| 83 |
+
"visual_family_oracle": 0.8805924097500771
|
| 84 |
+
},
|
| 85 |
+
"behavior_targets": {
|
| 86 |
+
"samples": 467,
|
| 87 |
+
"exact_oracle": 0.37687366167023556,
|
| 88 |
+
"visual_family_oracle": 0.7516059957173448
|
| 89 |
+
}
|
| 90 |
+
},
|
| 91 |
+
"official_test": {
|
| 92 |
+
"all_supported": {
|
| 93 |
+
"samples": 5921,
|
| 94 |
+
"exact_top1": 0.6302989359905421,
|
| 95 |
+
"visual_family_top1": 0.6818105049822665
|
| 96 |
+
},
|
| 97 |
+
"behavior_targets": {
|
| 98 |
+
"samples": 635,
|
| 99 |
+
"exact_top1": 0.3543307086614173,
|
| 100 |
+
"visual_family_top1": 0.5574803149606299
|
| 101 |
+
}
|
| 102 |
+
},
|
| 103 |
+
"track": "R_noncommercial_only",
|
| 104 |
+
"product_validation": false,
|
| 105 |
+
"distillation_allowed": false
|
| 106 |
+
}
|
scripts/audit_math_ink_06_behavior_exact_gate.py
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""기존 행동 head를 teacher 형태군과 confidence gate로 결합해 실제 exact 회수율을 감사한다."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from datetime import datetime, timezone
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import sys
|
| 10 |
+
from typing import Any, Sequence
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
from torch.utils.data import DataLoader, TensorDataset
|
| 14 |
+
|
| 15 |
+
PROJECT_ROOT = Path(__file__).parents[1]
|
| 16 |
+
SOURCE_ROOT = PROJECT_ROOT / "src"
|
| 17 |
+
for path in (PROJECT_ROOT, SOURCE_ROOT):
|
| 18 |
+
if str(path) not in sys.path:
|
| 19 |
+
sys.path.insert(0, str(path))
|
| 20 |
+
|
| 21 |
+
from math_grid_drawer.research.behavior_role_head06 import BehaviorRoleHead06
|
| 22 |
+
from math_grid_drawer.research.trajectory_sequence import visual_label_family
|
| 23 |
+
from scripts.audit_math_ink_06_case_context import _load_model06
|
| 24 |
+
from scripts.crohme_lattice_common import writer_fit_validation
|
| 25 |
+
from scripts.train_crohme_segmentation_lattice_selector import _samples
|
| 26 |
+
from scripts.train_math_ink_06_behavior_role import _materialize_split06
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _parse_args() -> argparse.Namespace:
|
| 30 |
+
"""필요 변수: seed별 adapter/behavior head·CROHME 공식 split. 작동 원리: validation-only gate 감사 CLI를 만든다."""
|
| 31 |
+
|
| 32 |
+
parser = argparse.ArgumentParser(description="Audit Math Ink 0.6 behavior exact gate")
|
| 33 |
+
parser.add_argument("--adapter", type=Path, action="append", required=True)
|
| 34 |
+
parser.add_argument("--behavior-head", type=Path, action="append", required=True)
|
| 35 |
+
parser.add_argument(
|
| 36 |
+
"--train-root", type=Path,
|
| 37 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/trainData",
|
| 38 |
+
)
|
| 39 |
+
parser.add_argument(
|
| 40 |
+
"--test-root", type=Path,
|
| 41 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/testDataGT",
|
| 42 |
+
)
|
| 43 |
+
parser.add_argument("--profile", default="median_height_32")
|
| 44 |
+
parser.add_argument("--teacher-batch-size", type=int, default=256)
|
| 45 |
+
parser.add_argument("--batch-size", type=int, default=256)
|
| 46 |
+
parser.add_argument("--device", choices=("cuda", "cpu"), default="cuda")
|
| 47 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 48 |
+
args = parser.parse_args()
|
| 49 |
+
if len(args.adapter) != len(args.behavior_head):
|
| 50 |
+
raise ValueError("adapter와 behavior head 개수는 같아야 합니다.")
|
| 51 |
+
if not args.adapter:
|
| 52 |
+
raise ValueError("한 개 이상의 seed artifact가 필요합니다.")
|
| 53 |
+
return args
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _role_label06(truth_label: str, role_index: int) -> str | None:
|
| 57 |
+
"""필요 변수: target visual base·행동 role. 작동 원리: 같은 형태군 안에서만 exact label로 변환한다."""
|
| 58 |
+
|
| 59 |
+
base = "x" if truth_label == r"\times" else truth_label.lower()
|
| 60 |
+
if role_index == 0:
|
| 61 |
+
return base
|
| 62 |
+
if role_index == 1:
|
| 63 |
+
return base.upper()
|
| 64 |
+
if role_index == 2 and base == "x":
|
| 65 |
+
return r"\times"
|
| 66 |
+
return None
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def behavior_exact_gate_metrics06(
|
| 70 |
+
behavior_logits: torch.Tensor,
|
| 71 |
+
metadata: Sequence[dict[str, Any]],
|
| 72 |
+
*,
|
| 73 |
+
threshold: float,
|
| 74 |
+
) -> dict[str, float | int]:
|
| 75 |
+
"""필요 변수: role logit·teacher/truth metadata·threshold. 작동 원리: family 확인 뒤 rewrite하고 나머지는 abstain한다."""
|
| 76 |
+
|
| 77 |
+
if len(behavior_logits) != len(metadata):
|
| 78 |
+
raise ValueError("behavior logit과 metadata 길이가 다릅니다.")
|
| 79 |
+
probability = behavior_logits.softmax(dim=1)
|
| 80 |
+
confidence, roles = probability.max(dim=1)
|
| 81 |
+
teacher_correct = final_correct = family_correct = rewrites = beneficial = harmful = 0
|
| 82 |
+
abstained = 0
|
| 83 |
+
for index, row in enumerate(metadata):
|
| 84 |
+
truth = str(row["truth_label"])
|
| 85 |
+
teacher = str(row["teacher_label"])
|
| 86 |
+
teacher_hit = teacher == truth
|
| 87 |
+
family_hit = visual_label_family(teacher) == visual_label_family(truth)
|
| 88 |
+
selected = _role_label06(truth, int(roles[index]))
|
| 89 |
+
rewrite = (
|
| 90 |
+
family_hit
|
| 91 |
+
and selected is not None
|
| 92 |
+
and float(confidence[index]) >= threshold
|
| 93 |
+
)
|
| 94 |
+
final = selected if rewrite else teacher
|
| 95 |
+
teacher_correct += int(teacher_hit)
|
| 96 |
+
family_correct += int(family_hit)
|
| 97 |
+
final_correct += int(final == truth)
|
| 98 |
+
rewrites += int(rewrite)
|
| 99 |
+
abstained += int(not rewrite)
|
| 100 |
+
beneficial += int(rewrite and not teacher_hit and final == truth)
|
| 101 |
+
harmful += int(rewrite and teacher_hit and final != truth)
|
| 102 |
+
samples = len(metadata)
|
| 103 |
+
return {
|
| 104 |
+
"threshold": float(threshold),
|
| 105 |
+
"samples": samples,
|
| 106 |
+
"teacher_exact": teacher_correct / max(samples, 1),
|
| 107 |
+
"teacher_visual_family": family_correct / max(samples, 1),
|
| 108 |
+
"final_exact": final_correct / max(samples, 1),
|
| 109 |
+
"gain_pp": (final_correct - teacher_correct) * 100.0 / max(samples, 1),
|
| 110 |
+
"rewrites": rewrites,
|
| 111 |
+
"abstained": abstained,
|
| 112 |
+
"beneficial": beneficial,
|
| 113 |
+
"harmful": harmful,
|
| 114 |
+
"rewrite_precision": beneficial / max(beneficial + harmful, 1),
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _behavior_logits06(
|
| 119 |
+
checkpoint: Path,
|
| 120 |
+
dataset: TensorDataset,
|
| 121 |
+
*,
|
| 122 |
+
device: torch.device,
|
| 123 |
+
batch_size: int,
|
| 124 |
+
) -> torch.Tensor:
|
| 125 |
+
"""필요 변수: behavior checkpoint·raw dataset. 작동 원리: checkpoint fit 통계로 context를 정규화해 role logit을 반환한다."""
|
| 126 |
+
|
| 127 |
+
payload = torch.load(checkpoint, map_location="cpu", weights_only=False)
|
| 128 |
+
model = BehaviorRoleHead06(
|
| 129 |
+
sequence_channels=int(payload.get("sequence_channels", 19)),
|
| 130 |
+
context_features=len(payload["context_features"]),
|
| 131 |
+
hidden=int(payload["hidden"]),
|
| 132 |
+
dropout=float(payload["dropout"]),
|
| 133 |
+
).to(device)
|
| 134 |
+
model.load_state_dict(payload["state_dict"])
|
| 135 |
+
model.eval()
|
| 136 |
+
mean = payload["context_mean"].float()
|
| 137 |
+
scale = payload["context_scale"].float().clamp_min(1e-5)
|
| 138 |
+
rows = []
|
| 139 |
+
with torch.inference_mode():
|
| 140 |
+
for sequence, context, _target in DataLoader(
|
| 141 |
+
dataset, batch_size=batch_size, shuffle=False,
|
| 142 |
+
):
|
| 143 |
+
normalized = (context - mean) / scale
|
| 144 |
+
rows.append(model(sequence.to(device), normalized.to(device)).cpu())
|
| 145 |
+
return torch.cat(rows)
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _select_threshold06(
|
| 149 |
+
logits: torch.Tensor,
|
| 150 |
+
metadata: Sequence[dict[str, Any]],
|
| 151 |
+
) -> tuple[float, list[dict[str, float | int]]]:
|
| 152 |
+
"""필요 변수: validation role logit·metadata. 작동 원리: exact 우선·harm 최소·높은 threshold 순으로 gate를 고정한다."""
|
| 153 |
+
|
| 154 |
+
thresholds = tuple(index / 100.0 for index in range(0, 100, 2))
|
| 155 |
+
sweep = [
|
| 156 |
+
behavior_exact_gate_metrics06(logits, metadata, threshold=value)
|
| 157 |
+
for value in thresholds
|
| 158 |
+
]
|
| 159 |
+
selected = max(
|
| 160 |
+
sweep,
|
| 161 |
+
key=lambda row: (
|
| 162 |
+
float(row["final_exact"]),
|
| 163 |
+
-int(row["harmful"]),
|
| 164 |
+
float(row["threshold"]),
|
| 165 |
+
),
|
| 166 |
+
)
|
| 167 |
+
return float(selected["threshold"]), sweep
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def main() -> None:
|
| 171 |
+
"""필요 변수: 공식 train writer-validation과 held-out test. 작동 원리: seed별 threshold를 validation에서 잠그고 test에 한 번 적용한다."""
|
| 172 |
+
|
| 173 |
+
args = _parse_args()
|
| 174 |
+
if args.train_root.name.casefold() != "traindata" or args.test_root.name.casefold() != "testdatagt":
|
| 175 |
+
raise ValueError("CROHME2012 공식 trainData/testDataGT 조합만 허용합니다.")
|
| 176 |
+
device = torch.device(args.device)
|
| 177 |
+
if device.type == "cuda" and not torch.cuda.is_available():
|
| 178 |
+
raise RuntimeError("CUDA 감사를 요청했지만 사용할 수 없습니다.")
|
| 179 |
+
_fit, validation_samples = writer_fit_validation(args.train_root, args.profile)
|
| 180 |
+
test_samples = _samples(args.test_root, args.profile)
|
| 181 |
+
seed_rows = []
|
| 182 |
+
for adapter_path, head_path in zip(args.adapter, args.behavior_head, strict=True):
|
| 183 |
+
adapter_payload = torch.load(adapter_path, map_location="cpu", weights_only=False)
|
| 184 |
+
base_checkpoint = Path(str(adapter_payload["base_checkpoint"]))
|
| 185 |
+
if not base_checkpoint.is_absolute():
|
| 186 |
+
base_checkpoint = PROJECT_ROOT / base_checkpoint
|
| 187 |
+
engine, adapter = _load_model06(base_checkpoint, adapter_path, device)
|
| 188 |
+
validation, _validation_counts, validation_metadata = _materialize_split06(
|
| 189 |
+
validation_samples, engine, adapter, device=device,
|
| 190 |
+
teacher_batch_size=args.teacher_batch_size, return_metadata=True,
|
| 191 |
+
)
|
| 192 |
+
testing, _test_counts, test_metadata = _materialize_split06(
|
| 193 |
+
test_samples, engine, adapter, device=device,
|
| 194 |
+
teacher_batch_size=args.teacher_batch_size, return_metadata=True,
|
| 195 |
+
)
|
| 196 |
+
validation_logits = _behavior_logits06(
|
| 197 |
+
head_path, validation, device=device, batch_size=args.batch_size,
|
| 198 |
+
)
|
| 199 |
+
selected_threshold, sweep = _select_threshold06(
|
| 200 |
+
validation_logits, validation_metadata,
|
| 201 |
+
)
|
| 202 |
+
test_logits = _behavior_logits06(
|
| 203 |
+
head_path, testing, device=device, batch_size=args.batch_size,
|
| 204 |
+
)
|
| 205 |
+
seed_rows.append({
|
| 206 |
+
"adapter": str(adapter_path),
|
| 207 |
+
"behavior_head": str(head_path),
|
| 208 |
+
"selected_threshold": selected_threshold,
|
| 209 |
+
"validation_selected": behavior_exact_gate_metrics06(
|
| 210 |
+
validation_logits, validation_metadata, threshold=selected_threshold,
|
| 211 |
+
),
|
| 212 |
+
"validation_sweep": sweep,
|
| 213 |
+
"official_test": behavior_exact_gate_metrics06(
|
| 214 |
+
test_logits, test_metadata, threshold=selected_threshold,
|
| 215 |
+
),
|
| 216 |
+
})
|
| 217 |
+
del engine, adapter
|
| 218 |
+
if device.type == "cuda":
|
| 219 |
+
torch.cuda.empty_cache()
|
| 220 |
+
metric_names = (
|
| 221 |
+
"teacher_exact", "teacher_visual_family", "final_exact", "gain_pp",
|
| 222 |
+
"rewrite_precision",
|
| 223 |
+
)
|
| 224 |
+
summary = {
|
| 225 |
+
name: {
|
| 226 |
+
"values": [float(row["official_test"][name]) for row in seed_rows],
|
| 227 |
+
"mean": sum(float(row["official_test"][name]) for row in seed_rows) / len(seed_rows),
|
| 228 |
+
}
|
| 229 |
+
for name in metric_names
|
| 230 |
+
}
|
| 231 |
+
report = {
|
| 232 |
+
"experiment": "R-MATH-INK-06-BEHAVIOR-EXACT-GATE-001",
|
| 233 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 234 |
+
"split_contract": "CROHME trainData writer-validation threshold; testDataGT one-shot",
|
| 235 |
+
"scope": "truth symbol grouping conditional; c/C, x/X/times, z/Z only",
|
| 236 |
+
"seeds": seed_rows,
|
| 237 |
+
"official_test_summary": summary,
|
| 238 |
+
"track": "R_noncommercial_only",
|
| 239 |
+
"product_validation": False,
|
| 240 |
+
"distillation_allowed": False,
|
| 241 |
+
}
|
| 242 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 243 |
+
args.output.write_text(
|
| 244 |
+
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
|
| 245 |
+
encoding="utf-8",
|
| 246 |
+
)
|
| 247 |
+
print(json.dumps({
|
| 248 |
+
"official_test_summary": summary,
|
| 249 |
+
"thresholds": [row["selected_threshold"] for row in seed_rows],
|
| 250 |
+
"product_validation": False,
|
| 251 |
+
}, ensure_ascii=False, indent=2))
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
if __name__ == "__main__":
|
| 255 |
+
main()
|
scripts/audit_math_ink_06_formula_context_contract.py
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""연속 수식 group의 baseline 채널 주입 방식을 validation에서 선택하고 official test에 고정 적용한다."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from datetime import datetime, timezone
|
| 7 |
+
import json
|
| 8 |
+
import math
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import sys
|
| 11 |
+
from typing import Any, Sequence
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
PROJECT_ROOT = Path(__file__).parents[1]
|
| 16 |
+
SOURCE_ROOT = PROJECT_ROOT / "src"
|
| 17 |
+
for path in (PROJECT_ROOT, SOURCE_ROOT):
|
| 18 |
+
if str(path) not in sys.path:
|
| 19 |
+
sys.path.insert(0, str(path))
|
| 20 |
+
|
| 21 |
+
from math_grid_drawer.research.trajectory_sequence import visual_label_family
|
| 22 |
+
from scripts.audit_math_ink_06_case_context import _load_model06
|
| 23 |
+
from scripts.crohme_lattice_common import writer_fit_validation
|
| 24 |
+
from scripts.train_crohme_segmentation_lattice_selector import _samples
|
| 25 |
+
from scripts.train_math_ink_06_behavior_role import (
|
| 26 |
+
TARGET_LABELS06,
|
| 27 |
+
_canonical_group06,
|
| 28 |
+
_formula_box06,
|
| 29 |
+
)
|
| 30 |
+
from scripts.train_math_ink_06_skeleton_adapter import _fused_exact_family_logits06
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
CONTEXT_MODES06 = ("formula", "missing", "local_full", "height_only")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _parse_args() -> argparse.Namespace:
|
| 37 |
+
"""필요 변수: seed별 adapter·CROHME 공식 split. 작동 원리: validation-only context 계약 선택 CLI를 만든다."""
|
| 38 |
+
|
| 39 |
+
parser = argparse.ArgumentParser(description="Audit Math Ink 0.6 formula context contract")
|
| 40 |
+
parser.add_argument("--adapter", type=Path, action="append", required=True)
|
| 41 |
+
parser.add_argument(
|
| 42 |
+
"--train-root", type=Path,
|
| 43 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/trainData",
|
| 44 |
+
)
|
| 45 |
+
parser.add_argument(
|
| 46 |
+
"--test-root", type=Path,
|
| 47 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/testDataGT",
|
| 48 |
+
)
|
| 49 |
+
parser.add_argument("--profile", default="median_height_32")
|
| 50 |
+
parser.add_argument("--batch-size", type=int, default=256)
|
| 51 |
+
parser.add_argument("--device", choices=("cuda", "cpu"), default="cuda")
|
| 52 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 53 |
+
return parser.parse_args()
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def apply_formula_context_mode06(features: torch.Tensor, mode: str) -> torch.Tensor:
|
| 57 |
+
"""필요 변수: N×128×19 canonical feature·mode. 작동 원리: glyph shape는 보존하고 baseline 5채널만 교체한다."""
|
| 58 |
+
|
| 59 |
+
if mode not in CONTEXT_MODES06:
|
| 60 |
+
raise ValueError(f"지원하지 않는 formula context mode입니다: {mode}")
|
| 61 |
+
output = features.clone()
|
| 62 |
+
if mode == "formula":
|
| 63 |
+
return output
|
| 64 |
+
if mode == "missing":
|
| 65 |
+
output[..., 10:15] = 0.0
|
| 66 |
+
elif mode == "local_full":
|
| 67 |
+
output[..., 10] = 0.0
|
| 68 |
+
output[..., 11] = 1.0
|
| 69 |
+
output[..., 12] = 1.0
|
| 70 |
+
output[..., 13] = 0.5
|
| 71 |
+
output[..., 14] = 1.0
|
| 72 |
+
else:
|
| 73 |
+
height = output[..., 12].clone()
|
| 74 |
+
output[..., 10] = 0.0
|
| 75 |
+
output[..., 11] = 0.0
|
| 76 |
+
output[..., 12] = height
|
| 77 |
+
output[..., 13] = 0.0
|
| 78 |
+
output[..., 14] = 1.0
|
| 79 |
+
return output
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _materialize_formula_groups06(
|
| 83 |
+
samples: Sequence[dict[str, Any]],
|
| 84 |
+
labels: Sequence[str],
|
| 85 |
+
) -> tuple[torch.Tensor, list[str], torch.Tensor]:
|
| 86 |
+
"""필요 변수: 공식 formula·model vocabulary. 작동 원리: 정답 group별 sequence와 전체/행동대상 mask를 만든다."""
|
| 87 |
+
|
| 88 |
+
allowed = set(str(label) for label in labels)
|
| 89 |
+
sequences = []
|
| 90 |
+
truths: list[str] = []
|
| 91 |
+
target_mask = []
|
| 92 |
+
for formula in samples:
|
| 93 |
+
formula_box = _formula_box06(formula["strokes"])
|
| 94 |
+
for group, label_value in zip(
|
| 95 |
+
formula["truth_groups"], formula["truth_labels"], strict=True,
|
| 96 |
+
):
|
| 97 |
+
label = str(label_value)
|
| 98 |
+
if label not in allowed:
|
| 99 |
+
continue
|
| 100 |
+
sequences.append(_canonical_group06(group, formula["strokes"], formula_box))
|
| 101 |
+
truths.append(label)
|
| 102 |
+
target_mask.append(label in TARGET_LABELS06)
|
| 103 |
+
if not sequences:
|
| 104 |
+
raise ValueError("지원되는 formula symbol group이 없습니다.")
|
| 105 |
+
return torch.stack(sequences), truths, torch.tensor(target_mask, dtype=torch.bool)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def formula_context_metrics06(
|
| 109 |
+
logits: torch.Tensor,
|
| 110 |
+
truths: Sequence[str],
|
| 111 |
+
labels: Sequence[str],
|
| 112 |
+
target_mask: torch.Tensor,
|
| 113 |
+
) -> dict[str, dict[str, float | int]]:
|
| 114 |
+
"""필요 변수: exact logit·정답 label·target mask. 작동 원리: 전체와 행동 형태군 slice를 같은 방식으로 평가한다."""
|
| 115 |
+
|
| 116 |
+
if len(logits) != len(truths) or len(target_mask) != len(truths):
|
| 117 |
+
raise ValueError("formula context 평가 분모가 서로 다릅니다.")
|
| 118 |
+
predictions = logits.argmax(dim=1).tolist()
|
| 119 |
+
predicted_labels = [str(labels[index]) for index in predictions]
|
| 120 |
+
|
| 121 |
+
def summarize(indices: list[int]) -> dict[str, float | int]:
|
| 122 |
+
"""필요 변수: slice index. 작동 원리: exact와 visual-family top-1을 반환한다."""
|
| 123 |
+
|
| 124 |
+
exact = sum(predicted_labels[index] == truths[index] for index in indices)
|
| 125 |
+
family = sum(
|
| 126 |
+
visual_label_family(predicted_labels[index])
|
| 127 |
+
== visual_label_family(str(truths[index]))
|
| 128 |
+
for index in indices
|
| 129 |
+
)
|
| 130 |
+
return {
|
| 131 |
+
"samples": len(indices),
|
| 132 |
+
"exact_top1": exact / max(len(indices), 1),
|
| 133 |
+
"visual_family_top1": family / max(len(indices), 1),
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
all_indices = list(range(len(truths)))
|
| 137 |
+
target_indices = target_mask.nonzero(as_tuple=False).flatten().tolist()
|
| 138 |
+
return {
|
| 139 |
+
"all_supported": summarize(all_indices),
|
| 140 |
+
"behavior_targets": summarize(target_indices),
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def formula_context_branch_oracle06(
|
| 145 |
+
logits_by_mode: Sequence[torch.Tensor],
|
| 146 |
+
truths: Sequence[str],
|
| 147 |
+
labels: Sequence[str],
|
| 148 |
+
target_mask: torch.Tensor,
|
| 149 |
+
) -> dict[str, dict[str, float | int]]:
|
| 150 |
+
"""필요 변수: mode별 logit·정답. 작동 원리: 어느 branch든 맞힌 비배포 상한을 전체/행동 slice로 계산한다."""
|
| 151 |
+
|
| 152 |
+
if not logits_by_mode or any(len(logits) != len(truths) for logits in logits_by_mode):
|
| 153 |
+
raise ValueError("context branch oracle 분모가 올바르지 않습니다.")
|
| 154 |
+
predictions = [logits.argmax(dim=1).tolist() for logits in logits_by_mode]
|
| 155 |
+
|
| 156 |
+
def summarize(indices: list[int]) -> dict[str, float | int]:
|
| 157 |
+
"""필요 변수: slice index. 작동 원리: exact/visual-family branch oracle을 반환한다."""
|
| 158 |
+
|
| 159 |
+
exact = family = 0
|
| 160 |
+
for index in indices:
|
| 161 |
+
truth = str(truths[index])
|
| 162 |
+
predicted_labels = [str(labels[row[index]]) for row in predictions]
|
| 163 |
+
exact += int(any(label == truth for label in predicted_labels))
|
| 164 |
+
family += int(any(
|
| 165 |
+
visual_label_family(label) == visual_label_family(truth)
|
| 166 |
+
for label in predicted_labels
|
| 167 |
+
))
|
| 168 |
+
return {
|
| 169 |
+
"samples": len(indices),
|
| 170 |
+
"exact_oracle": exact / max(len(indices), 1),
|
| 171 |
+
"visual_family_oracle": family / max(len(indices), 1),
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
return {
|
| 175 |
+
"all_supported": summarize(list(range(len(truths)))),
|
| 176 |
+
"behavior_targets": summarize(
|
| 177 |
+
target_mask.nonzero(as_tuple=False).flatten().tolist(),
|
| 178 |
+
),
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def _ensemble_logits06(
|
| 183 |
+
features: torch.Tensor,
|
| 184 |
+
adapters: Sequence[Path],
|
| 185 |
+
*,
|
| 186 |
+
mode: str,
|
| 187 |
+
device: torch.device,
|
| 188 |
+
batch_size: int,
|
| 189 |
+
) -> tuple[torch.Tensor, tuple[str, ...]]:
|
| 190 |
+
"""필요 변수: formula feature·seed adapter·context mode. 작동 원리: seed log-probability ensemble을 계산한다."""
|
| 191 |
+
|
| 192 |
+
transformed = apply_formula_context_mode06(features, mode)
|
| 193 |
+
seed_rows = []
|
| 194 |
+
canonical_labels: tuple[str, ...] | None = None
|
| 195 |
+
for adapter_path in adapters:
|
| 196 |
+
payload = torch.load(adapter_path, map_location="cpu", weights_only=False)
|
| 197 |
+
base_checkpoint = Path(str(payload["base_checkpoint"]))
|
| 198 |
+
if not base_checkpoint.is_absolute():
|
| 199 |
+
base_checkpoint = PROJECT_ROOT / base_checkpoint
|
| 200 |
+
engine, adapter = _load_model06(base_checkpoint, adapter_path, device)
|
| 201 |
+
labels = tuple(str(label) for label in engine.labels)
|
| 202 |
+
if canonical_labels is not None and labels != canonical_labels:
|
| 203 |
+
raise ValueError("seed별 vocabulary 순서가 다릅니다.")
|
| 204 |
+
canonical_labels = labels
|
| 205 |
+
rows = []
|
| 206 |
+
engine.model.eval()
|
| 207 |
+
adapter.eval()
|
| 208 |
+
with torch.inference_mode():
|
| 209 |
+
for start in range(0, len(transformed), batch_size):
|
| 210 |
+
batch = transformed[start:start + batch_size].to(device)
|
| 211 |
+
hypotheses = batch.unsqueeze(1).expand(-1, 4, -1, -1).contiguous()
|
| 212 |
+
exact, _family = _fused_exact_family_logits06(
|
| 213 |
+
engine, adapter, hypotheses,
|
| 214 |
+
)
|
| 215 |
+
rows.append(exact.cpu().log_softmax(dim=1))
|
| 216 |
+
seed_rows.append(torch.cat(rows))
|
| 217 |
+
del engine, adapter
|
| 218 |
+
if device.type == "cuda":
|
| 219 |
+
torch.cuda.empty_cache()
|
| 220 |
+
assert canonical_labels is not None
|
| 221 |
+
ensemble = torch.logsumexp(torch.stack(seed_rows), dim=0) - math.log(len(seed_rows))
|
| 222 |
+
return ensemble, canonical_labels
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def main() -> None:
|
| 226 |
+
"""필요 변수: writer-validation과 official test. 작동 원리: context mode를 validation에서 잠그고 test에는 선택 mode만 적용한다."""
|
| 227 |
+
|
| 228 |
+
args = _parse_args()
|
| 229 |
+
if args.train_root.name.casefold() != "traindata" or args.test_root.name.casefold() != "testdatagt":
|
| 230 |
+
raise ValueError("CROHME2012 공식 trainData/testDataGT 조합만 허용합니다.")
|
| 231 |
+
device = torch.device(args.device)
|
| 232 |
+
if device.type == "cuda" and not torch.cuda.is_available():
|
| 233 |
+
raise RuntimeError("CUDA 감사를 요청했지만 사용할 수 없습니다.")
|
| 234 |
+
first_payload = torch.load(args.adapter[0], map_location="cpu", weights_only=False)
|
| 235 |
+
first_base = Path(str(first_payload["base_checkpoint"]))
|
| 236 |
+
if not first_base.is_absolute():
|
| 237 |
+
first_base = PROJECT_ROOT / first_base
|
| 238 |
+
first_engine, _first_adapter = _load_model06(first_base, args.adapter[0], device)
|
| 239 |
+
labels = first_engine.labels
|
| 240 |
+
del first_engine, _first_adapter
|
| 241 |
+
if device.type == "cuda":
|
| 242 |
+
torch.cuda.empty_cache()
|
| 243 |
+
_fit, validation_samples = writer_fit_validation(args.train_root, args.profile)
|
| 244 |
+
test_samples = _samples(args.test_root, args.profile)
|
| 245 |
+
validation_features, validation_truths, validation_target = _materialize_formula_groups06(
|
| 246 |
+
validation_samples, labels,
|
| 247 |
+
)
|
| 248 |
+
validation_rows = []
|
| 249 |
+
validation_logits_by_mode = []
|
| 250 |
+
for mode in CONTEXT_MODES06:
|
| 251 |
+
logits, current_labels = _ensemble_logits06(
|
| 252 |
+
validation_features, args.adapter, mode=mode,
|
| 253 |
+
device=device, batch_size=args.batch_size,
|
| 254 |
+
)
|
| 255 |
+
validation_rows.append({
|
| 256 |
+
"mode": mode,
|
| 257 |
+
**formula_context_metrics06(
|
| 258 |
+
logits, validation_truths, current_labels, validation_target,
|
| 259 |
+
),
|
| 260 |
+
})
|
| 261 |
+
validation_logits_by_mode.append(logits)
|
| 262 |
+
selected = max(
|
| 263 |
+
validation_rows,
|
| 264 |
+
key=lambda row: (
|
| 265 |
+
float(row["behavior_targets"]["visual_family_top1"]),
|
| 266 |
+
float(row["all_supported"]["visual_family_top1"]),
|
| 267 |
+
float(row["all_supported"]["exact_top1"]),
|
| 268 |
+
row["mode"] == "formula",
|
| 269 |
+
),
|
| 270 |
+
)
|
| 271 |
+
test_features, test_truths, test_target = _materialize_formula_groups06(
|
| 272 |
+
test_samples, labels,
|
| 273 |
+
)
|
| 274 |
+
test_logits, test_labels = _ensemble_logits06(
|
| 275 |
+
test_features, args.adapter, mode=str(selected["mode"]),
|
| 276 |
+
device=device, batch_size=args.batch_size,
|
| 277 |
+
)
|
| 278 |
+
report = {
|
| 279 |
+
"experiment": "R-MATH-INK-06-FORMULA-CONTEXT-CONTRACT-001",
|
| 280 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 281 |
+
"split_contract": "CROHME trainData writer-validation mode selection; testDataGT one-shot",
|
| 282 |
+
"context_modes": CONTEXT_MODES06,
|
| 283 |
+
"validation_sweep": validation_rows,
|
| 284 |
+
"selected_mode": str(selected["mode"]),
|
| 285 |
+
"selected_validation": selected,
|
| 286 |
+
"validation_branch_oracle": formula_context_branch_oracle06(
|
| 287 |
+
validation_logits_by_mode,
|
| 288 |
+
validation_truths,
|
| 289 |
+
labels,
|
| 290 |
+
validation_target,
|
| 291 |
+
),
|
| 292 |
+
"official_test": formula_context_metrics06(
|
| 293 |
+
test_logits, test_truths, test_labels, test_target,
|
| 294 |
+
),
|
| 295 |
+
"track": "R_noncommercial_only",
|
| 296 |
+
"product_validation": False,
|
| 297 |
+
"distillation_allowed": False,
|
| 298 |
+
}
|
| 299 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 300 |
+
args.output.write_text(
|
| 301 |
+
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
|
| 302 |
+
encoding="utf-8",
|
| 303 |
+
)
|
| 304 |
+
print(json.dumps(report, ensure_ascii=False, indent=2))
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
if __name__ == "__main__":
|
| 308 |
+
main()
|
scripts/summarize_math_ink_06_formula_adapter_seeds.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Formula adapter seed 17·31·47 결과를 동일 gate로 요약한다."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from datetime import datetime, timezone
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from statistics import mean, pstdev
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _metric06(reports: list[dict], section: str, name: str) -> dict:
|
| 13 |
+
"""필요 변수: seed report·section·metric. 작동 원리: 값·평균·표준편차·최솟값을 반환한다."""
|
| 14 |
+
|
| 15 |
+
values = [float(report[section][name]) for report in reports]
|
| 16 |
+
return {
|
| 17 |
+
"values": values,
|
| 18 |
+
"mean": mean(values),
|
| 19 |
+
"std": pstdev(values),
|
| 20 |
+
"minimum": min(values),
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def main() -> None:
|
| 25 |
+
"""필요 변수: seed report 세 개. 작동 원리: validation/test 92% gate와 일반화 gap을 고정한다."""
|
| 26 |
+
|
| 27 |
+
parser = argparse.ArgumentParser(description="Summarize Math Ink 0.6 formula adapters")
|
| 28 |
+
parser.add_argument("--report", type=Path, action="append", required=True)
|
| 29 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 30 |
+
args = parser.parse_args()
|
| 31 |
+
reports = [
|
| 32 |
+
json.loads(path.read_text(encoding="utf-8")) for path in args.report
|
| 33 |
+
]
|
| 34 |
+
if len(reports) != 3:
|
| 35 |
+
raise ValueError("formula adapter 요약에는 seed report 세 개가 필요합니다.")
|
| 36 |
+
validation_visual = _metric06(
|
| 37 |
+
reports, "selected_validation", "visual_family_top1",
|
| 38 |
+
)
|
| 39 |
+
test_visual = _metric06(reports, "official_test", "visual_family_top1")
|
| 40 |
+
summary = {
|
| 41 |
+
"experiment": "R-MATH-INK-06-FORMULA-ADAPTER-3SEED-001",
|
| 42 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 43 |
+
"reports": [str(path) for path in args.report],
|
| 44 |
+
"metrics": {
|
| 45 |
+
"validation_exact_top1": _metric06(
|
| 46 |
+
reports, "selected_validation", "exact_top1",
|
| 47 |
+
),
|
| 48 |
+
"validation_family_head_top1": _metric06(
|
| 49 |
+
reports, "selected_validation", "family_head_top1",
|
| 50 |
+
),
|
| 51 |
+
"validation_visual_family_top1": validation_visual,
|
| 52 |
+
"official_test_exact_top1": _metric06(
|
| 53 |
+
reports, "official_test", "exact_top1",
|
| 54 |
+
),
|
| 55 |
+
"official_test_family_head_top1": _metric06(
|
| 56 |
+
reports, "official_test", "family_head_top1",
|
| 57 |
+
),
|
| 58 |
+
"official_test_visual_family_top1": test_visual,
|
| 59 |
+
"visual_family_generalization_gap_pp": {
|
| 60 |
+
"values": [
|
| 61 |
+
(
|
| 62 |
+
float(report["selected_validation"]["visual_family_top1"])
|
| 63 |
+
- float(report["official_test"]["visual_family_top1"])
|
| 64 |
+
) * 100.0
|
| 65 |
+
for report in reports
|
| 66 |
+
],
|
| 67 |
+
},
|
| 68 |
+
},
|
| 69 |
+
"decision": {
|
| 70 |
+
"all_seed_validation_92_passed": validation_visual["minimum"] >= 0.92,
|
| 71 |
+
"all_seed_official_test_92_passed": test_visual["minimum"] >= 0.92,
|
| 72 |
+
"architecture_recoverable": validation_visual["minimum"] >= 0.92,
|
| 73 |
+
"writer_domain_generalization_gate_passed": test_visual["minimum"] >= 0.92,
|
| 74 |
+
"product_validation": False,
|
| 75 |
+
"distillation_allowed": False,
|
| 76 |
+
"next_gate": "P 연속식 writer/device-disjoint formula adapter 학습·평가",
|
| 77 |
+
},
|
| 78 |
+
"track": "R_noncommercial_only",
|
| 79 |
+
"product_validation": False,
|
| 80 |
+
}
|
| 81 |
+
gaps = summary["metrics"]["visual_family_generalization_gap_pp"]["values"]
|
| 82 |
+
summary["metrics"]["visual_family_generalization_gap_pp"].update({
|
| 83 |
+
"mean": mean(gaps),
|
| 84 |
+
"std": pstdev(gaps),
|
| 85 |
+
"maximum": max(gaps),
|
| 86 |
+
})
|
| 87 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 88 |
+
args.output.write_text(
|
| 89 |
+
json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
|
| 90 |
+
encoding="utf-8",
|
| 91 |
+
)
|
| 92 |
+
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
if __name__ == "__main__":
|
| 96 |
+
main()
|
scripts/train_math_ink_06_behavior_role.py
ADDED
|
@@ -0,0 +1,861 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CROHME 연속 수식에서 AIFlow 0.6 행동 의미 역할 head를 GPU 학습한다."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from collections import Counter
|
| 7 |
+
from copy import deepcopy
|
| 8 |
+
from datetime import datetime, timezone
|
| 9 |
+
import json
|
| 10 |
+
import math
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
import random
|
| 13 |
+
import sys
|
| 14 |
+
from typing import Any, Sequence
|
| 15 |
+
|
| 16 |
+
import numpy as np
|
| 17 |
+
import torch
|
| 18 |
+
from torch import Tensor
|
| 19 |
+
from torch.utils.data import DataLoader, TensorDataset
|
| 20 |
+
|
| 21 |
+
PROJECT_ROOT = Path(__file__).parents[1]
|
| 22 |
+
SOURCE_ROOT = PROJECT_ROOT / "src"
|
| 23 |
+
for path in (PROJECT_ROOT, SOURCE_ROOT):
|
| 24 |
+
if str(path) not in sys.path:
|
| 25 |
+
sys.path.insert(0, str(path))
|
| 26 |
+
|
| 27 |
+
from math_grid_drawer.research.behavior_role_head06 import (
|
| 28 |
+
BEHAVIOR_CONTEXT_FEATURES06,
|
| 29 |
+
BEHAVIOR_ROLE_LABELS06,
|
| 30 |
+
BehaviorRoleHead06,
|
| 31 |
+
behavior_role_index06,
|
| 32 |
+
validate_behavior_context06,
|
| 33 |
+
)
|
| 34 |
+
from math_grid_drawer.research.cross_visual import CROSS_FEATURE_NAMES, cross_pair_feature_rows
|
| 35 |
+
from math_grid_drawer.research.ink06_canonical import canonicalize_ink06
|
| 36 |
+
from scripts.crohme_lattice_common import writer_fit_validation
|
| 37 |
+
from scripts.train_crohme_segmentation_lattice_selector import _samples
|
| 38 |
+
from scripts.train_math_ink_06_skeleton_adapter import (
|
| 39 |
+
_fused_exact_family_logits06,
|
| 40 |
+
_paired_record_feature06,
|
| 41 |
+
)
|
| 42 |
+
from scripts.audit_math_ink_06_case_context import _load_model06
|
| 43 |
+
from scripts.materialize_math_ink_06_paired_feature_cache import _selected_records
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
TARGET_LABELS06 = frozenset({"c", "C", "x", "X", "z", "Z", r"\times"})
|
| 47 |
+
OPERATOR_LABELS06 = frozenset({
|
| 48 |
+
"+", "-", "=", "<", ">", "/", "*", r"\times", r"\div", r"\pm", r"\cdot",
|
| 49 |
+
})
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _parse_args() -> argparse.Namespace:
|
| 53 |
+
"""필요 변수: adapter·공식 CROHME root·GPU 설정. 작동 원리: seed별 재현 가능한 행동학습 CLI를 만든다."""
|
| 54 |
+
|
| 55 |
+
parser = argparse.ArgumentParser(description="Train Math Ink 0.6 behavior role head")
|
| 56 |
+
parser.add_argument("--adapter", type=Path, required=True)
|
| 57 |
+
parser.add_argument(
|
| 58 |
+
"--train-root", type=Path,
|
| 59 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/trainData",
|
| 60 |
+
)
|
| 61 |
+
parser.add_argument(
|
| 62 |
+
"--test-root", type=Path,
|
| 63 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/testDataGT",
|
| 64 |
+
)
|
| 65 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 66 |
+
parser.add_argument("--seed", type=int, required=True)
|
| 67 |
+
parser.add_argument("--epochs", type=int, default=60)
|
| 68 |
+
parser.add_argument("--patience", type=int, default=14)
|
| 69 |
+
parser.add_argument("--batch-size", type=int, default=256)
|
| 70 |
+
parser.add_argument("--teacher-batch-size", type=int, default=256)
|
| 71 |
+
parser.add_argument("--learning-rate", type=float, default=1e-3)
|
| 72 |
+
parser.add_argument("--weight-decay", type=float, default=1e-3)
|
| 73 |
+
parser.add_argument("--hidden", type=int, default=48)
|
| 74 |
+
parser.add_argument("--dropout", type=float, default=0.15)
|
| 75 |
+
parser.add_argument("--profile", default="median_height_32")
|
| 76 |
+
parser.add_argument("--device", choices=("cuda", "cpu"), default="cuda")
|
| 77 |
+
parser.add_argument("--product-proxy-per-label", type=int, default=0)
|
| 78 |
+
parser.add_argument("--product-proxy-loss-weight", type=float, default=0.35)
|
| 79 |
+
parser.add_argument("--product-proxy-target-bases", default="cosuvwxz")
|
| 80 |
+
parser.add_argument("--product-proxy-lowercase-ratio", type=float, default=1.0)
|
| 81 |
+
parser.add_argument(
|
| 82 |
+
"--data", type=Path,
|
| 83 |
+
default=PROJECT_ROOT / "research/data/open_pretrain/hwrt_expanded_v2/hwrt_expanded.jsonl.gz",
|
| 84 |
+
)
|
| 85 |
+
parser.add_argument(
|
| 86 |
+
"--commercial-paired", type=Path,
|
| 87 |
+
default=PROJECT_ROOT / "research/data/external_trajectory_v1/commercial_ccby4.jsonl.gz",
|
| 88 |
+
)
|
| 89 |
+
parser.add_argument(
|
| 90 |
+
"--dataset-registry", type=Path, default=PROJECT_ROOT / "research/dataset_registry.json",
|
| 91 |
+
)
|
| 92 |
+
parser.add_argument(
|
| 93 |
+
"--source-registry", type=Path,
|
| 94 |
+
default=PROJECT_ROOT / "research/math_ink_06_source_registry.json",
|
| 95 |
+
)
|
| 96 |
+
parser.add_argument(
|
| 97 |
+
"--hwrt-approval", type=Path,
|
| 98 |
+
default=PROJECT_ROOT / "research/approvals/HWRT-ODBL-USE-APPROVAL-v1.json",
|
| 99 |
+
)
|
| 100 |
+
return parser.parse_args()
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _seed06(seed: int) -> None:
|
| 104 |
+
"""필요 변수: 실험 seed. 작동 원리: Python·NumPy·PyTorch 초기화를 함께 고정한다."""
|
| 105 |
+
|
| 106 |
+
random.seed(seed)
|
| 107 |
+
np.random.seed(seed)
|
| 108 |
+
torch.manual_seed(seed)
|
| 109 |
+
if torch.cuda.is_available():
|
| 110 |
+
torch.cuda.manual_seed_all(seed)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def _group_box06(group: frozenset[int], strokes: Sequence[dict[str, Any]]) -> dict[str, float]:
|
| 114 |
+
"""필요 변수: 정답 stroke group·원본 획. 작동 원리: 수식 좌표계의 bbox와 중심을 계산한다."""
|
| 115 |
+
|
| 116 |
+
points = [point for index in group for point in strokes[index]["points"]]
|
| 117 |
+
xs, ys = [float(point[0]) for point in points], [float(point[1]) for point in points]
|
| 118 |
+
left, top, right, bottom = min(xs), min(ys), max(xs), max(ys)
|
| 119 |
+
return {
|
| 120 |
+
"left": left, "top": top, "right": right, "bottom": bottom,
|
| 121 |
+
"width": max(right - left, 1e-5), "height": max(bottom - top, 1e-5),
|
| 122 |
+
"cx": (left + right) * 0.5, "cy": (top + bottom) * 0.5,
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _formula_box06(strokes: Sequence[dict[str, Any]]) -> dict[str, float]:
|
| 127 |
+
"""필요 변수: 한 수식의 전체 획. 작동 원리: canvas 정규화에 필요한 원점·크기를 반환한다."""
|
| 128 |
+
|
| 129 |
+
group = frozenset(range(len(strokes)))
|
| 130 |
+
return _group_box06(group, strokes)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _canonical_group06(
|
| 134 |
+
group: frozenset[int],
|
| 135 |
+
strokes: Sequence[dict[str, Any]],
|
| 136 |
+
formula_box: dict[str, float],
|
| 137 |
+
) -> Tensor:
|
| 138 |
+
"""필요 변수: symbol group·수식 bbox. 작동 원리: 원본 필순을 보존하며 formula-relative 128×19 입력을 만든다."""
|
| 139 |
+
|
| 140 |
+
selected = []
|
| 141 |
+
for order, stroke_index in enumerate(sorted(group)):
|
| 142 |
+
stroke = strokes[stroke_index]
|
| 143 |
+
selected.append({
|
| 144 |
+
"order": order,
|
| 145 |
+
"stroke_id": order,
|
| 146 |
+
"points": [
|
| 147 |
+
[
|
| 148 |
+
float(point[0]) - formula_box["left"],
|
| 149 |
+
float(point[1]) - formula_box["top"],
|
| 150 |
+
point[2] if len(point) > 2 else None,
|
| 151 |
+
]
|
| 152 |
+
for point in stroke["points"]
|
| 153 |
+
],
|
| 154 |
+
})
|
| 155 |
+
ink = canonicalize_ink06(
|
| 156 |
+
selected,
|
| 157 |
+
canvas_width=formula_box["width"],
|
| 158 |
+
canvas_height=formula_box["height"],
|
| 159 |
+
source_modality="online",
|
| 160 |
+
trust_timestamps=False,
|
| 161 |
+
)
|
| 162 |
+
return torch.from_numpy(ink.features)
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _teacher_role_indices06(labels: Sequence[str]) -> dict[str, list[int]]:
|
| 166 |
+
"""필요 변수: 378 vocabulary. 작동 원리: 이웃의 예측 확률을 digit/identifier/operator 역할로 합칠 index를 만든다."""
|
| 167 |
+
|
| 168 |
+
digit = [index for index, label in enumerate(labels) if str(label).isdigit()]
|
| 169 |
+
operator = [index for index, label in enumerate(labels) if str(label) in OPERATOR_LABELS06]
|
| 170 |
+
identifier = [
|
| 171 |
+
index for index, label in enumerate(labels)
|
| 172 |
+
if (
|
| 173 |
+
(len(str(label)) == 1 and str(label).isalpha())
|
| 174 |
+
or (str(label).startswith("\\") and str(label) not in OPERATOR_LABELS06)
|
| 175 |
+
)
|
| 176 |
+
]
|
| 177 |
+
return {"digit": digit, "identifier": identifier, "operator": operator}
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def _role_mass06(probability: Tensor, indices: dict[str, list[int]]) -> list[float]:
|
| 181 |
+
"""필요 변수: 한 기호 teacher 확률·역할 index. 작동 원리: 네 coarse role 확률을 반환한다."""
|
| 182 |
+
|
| 183 |
+
digit = float(probability[indices["digit"]].sum())
|
| 184 |
+
identifier = float(probability[indices["identifier"]].sum())
|
| 185 |
+
operator = float(probability[indices["operator"]].sum())
|
| 186 |
+
return [digit, identifier, operator, max(0.0, 1.0 - digit - identifier - operator)]
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def _neighbor06(
|
| 190 |
+
boxes: Sequence[dict[str, float]],
|
| 191 |
+
index: int,
|
| 192 |
+
*,
|
| 193 |
+
direction: str,
|
| 194 |
+
reference_height: float,
|
| 195 |
+
) -> int | None:
|
| 196 |
+
"""필요 변수: group bbox·기준 위치·방향. 작동 원리: 같은 로컬 행에 가까운 좌우 기호를 선택한다."""
|
| 197 |
+
|
| 198 |
+
target = boxes[index]
|
| 199 |
+
candidates = []
|
| 200 |
+
for other_index, other in enumerate(boxes):
|
| 201 |
+
if other_index == index:
|
| 202 |
+
continue
|
| 203 |
+
horizontal = other["cx"] < target["cx"] if direction == "left" else other["cx"] > target["cx"]
|
| 204 |
+
if not horizontal:
|
| 205 |
+
continue
|
| 206 |
+
vertical = abs(other["cy"] - target["cy"]) / max(reference_height, target["height"], 1e-5)
|
| 207 |
+
if vertical > 0.85:
|
| 208 |
+
continue
|
| 209 |
+
gap = (
|
| 210 |
+
max(0.0, target["left"] - other["right"])
|
| 211 |
+
if direction == "left"
|
| 212 |
+
else max(0.0, other["left"] - target["right"])
|
| 213 |
+
)
|
| 214 |
+
candidates.append((gap + vertical * reference_height * 0.35, other_index))
|
| 215 |
+
return min(candidates)[1] if candidates else None
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def _family_log_probability06(
|
| 219 |
+
log_probability: Tensor,
|
| 220 |
+
labels: Sequence[str],
|
| 221 |
+
base: str,
|
| 222 |
+
) -> tuple[float, float, float, float]:
|
| 223 |
+
"""필요 변수: teacher log 확률·case base. 작동 원리: lower/upper/times와 family 총질량을 공통 feature로 만든다."""
|
| 224 |
+
|
| 225 |
+
label_to_index = {str(label): index for index, label in enumerate(labels)}
|
| 226 |
+
members = (base, base.upper(), r"\times")
|
| 227 |
+
values = [
|
| 228 |
+
float(log_probability[label_to_index[label]]) if label in label_to_index else -20.0
|
| 229 |
+
for label in members
|
| 230 |
+
]
|
| 231 |
+
available = [label_to_index[label] for label in members if label in label_to_index]
|
| 232 |
+
mass = float(log_probability[available].logsumexp(dim=0).exp()) if available else 0.0
|
| 233 |
+
return values[0], values[1], values[2], mass
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def _context_row06(
|
| 237 |
+
formula: dict[str, Any],
|
| 238 |
+
index: int,
|
| 239 |
+
boxes: Sequence[dict[str, float]],
|
| 240 |
+
teacher_logits: Tensor,
|
| 241 |
+
labels: Sequence[str],
|
| 242 |
+
role_indices: dict[str, list[int]],
|
| 243 |
+
) -> list[float]:
|
| 244 |
+
"""필요 변수: 수식·target group·teacher logits. 작동 원리: 누수 없는 예측 이웃과 실제 배치·획 feature를 49차로 만든다."""
|
| 245 |
+
|
| 246 |
+
target_label = str(formula["truth_labels"][index])
|
| 247 |
+
base = "x" if target_label == r"\times" else target_label.lower()
|
| 248 |
+
probability = teacher_logits.softmax(dim=1)
|
| 249 |
+
log_probability = teacher_logits.log_softmax(dim=1)
|
| 250 |
+
target_probability = probability[index]
|
| 251 |
+
lower, upper, times, family_mass = _family_log_probability06(
|
| 252 |
+
log_probability[index], labels, base,
|
| 253 |
+
)
|
| 254 |
+
entropy = float(
|
| 255 |
+
-(target_probability * target_probability.clamp_min(1e-9).log()).sum()
|
| 256 |
+
/ max(math.log(len(labels)), 1.0)
|
| 257 |
+
)
|
| 258 |
+
heights = [box["height"] for box in boxes]
|
| 259 |
+
reference_height = float(np.median(heights))
|
| 260 |
+
widths = [box["width"] for box in boxes]
|
| 261 |
+
reference_width = float(np.median(widths))
|
| 262 |
+
left_index = _neighbor06(boxes, index, direction="left", reference_height=reference_height)
|
| 263 |
+
right_index = _neighbor06(boxes, index, direction="right", reference_height=reference_height)
|
| 264 |
+
left_role = _role_mass06(probability[left_index], role_indices) if left_index is not None else [0.0] * 4
|
| 265 |
+
right_role = _role_mass06(probability[right_index], role_indices) if right_index is not None else [0.0] * 4
|
| 266 |
+
target = boxes[index]
|
| 267 |
+
left = boxes[left_index] if left_index is not None else None
|
| 268 |
+
right = boxes[right_index] if right_index is not None else None
|
| 269 |
+
formula_box = _formula_box06(formula["strokes"])
|
| 270 |
+
group = formula["truth_groups"][index]
|
| 271 |
+
target_strokes = [formula["strokes"][stroke_index] for stroke_index in sorted(group)]
|
| 272 |
+
cross = np.zeros(len(CROSS_FEATURE_NAMES), dtype=np.float32)
|
| 273 |
+
if len(target_strokes) == 2:
|
| 274 |
+
cross = cross_pair_feature_rows([(0, 1)], target_strokes)[0]
|
| 275 |
+
point_count = sum(len(stroke.get("points") or []) for stroke in target_strokes)
|
| 276 |
+
values = [
|
| 277 |
+
lower, upper, times, family_mass,
|
| 278 |
+
float(target_probability.max()), entropy,
|
| 279 |
+
*left_role, *right_role,
|
| 280 |
+
max(0.0, target["left"] - left["right"]) / reference_height if left else 2.0,
|
| 281 |
+
max(0.0, right["left"] - target["right"]) / reference_height if right else 2.0,
|
| 282 |
+
(target["cy"] - left["cy"]) / reference_height if left else 0.0,
|
| 283 |
+
(right["cy"] - target["cy"]) / reference_height if right else 0.0,
|
| 284 |
+
(target["cx"] - formula_box["left"]) / formula_box["width"],
|
| 285 |
+
(target["cy"] - formula_box["top"]) / formula_box["height"],
|
| 286 |
+
target["width"] / formula_box["width"],
|
| 287 |
+
target["height"] / formula_box["height"],
|
| 288 |
+
target["height"] / max(reference_height, 1e-5),
|
| 289 |
+
target["width"] / max(reference_width, 1e-5),
|
| 290 |
+
min(len(group) / 8.0, 1.0),
|
| 291 |
+
min(point_count / 256.0, 1.0),
|
| 292 |
+
float(base == "c"), float(base == "x"), float(base == "z"),
|
| 293 |
+
*cross.tolist(),
|
| 294 |
+
float(len(target_strokes) == 2),
|
| 295 |
+
]
|
| 296 |
+
validate_behavior_context06(values)
|
| 297 |
+
return values
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
def _materialize_split06(
|
| 301 |
+
samples: Sequence[dict[str, Any]],
|
| 302 |
+
engine,
|
| 303 |
+
adapter: torch.nn.Module,
|
| 304 |
+
*,
|
| 305 |
+
device: torch.device,
|
| 306 |
+
teacher_batch_size: int,
|
| 307 |
+
return_metadata: bool = False,
|
| 308 |
+
) -> tuple[TensorDataset, dict[str, int]] | tuple[TensorDataset, dict[str, int], list[dict[str, Any]]]:
|
| 309 |
+
"""필요 변수: 수식 split·frozen teacher. 작동 원리: 모든 이웃은 teacher 예측으로 만들고 target stroke만 corpus에 보존한다."""
|
| 310 |
+
|
| 311 |
+
labels = tuple(str(label) for label in engine.labels)
|
| 312 |
+
role_indices = _teacher_role_indices06(labels)
|
| 313 |
+
sequence_rows: list[Tensor] = []
|
| 314 |
+
context_rows: list[list[float]] = []
|
| 315 |
+
target_rows: list[int] = []
|
| 316 |
+
metadata_rows: list[dict[str, Any]] = []
|
| 317 |
+
counts: Counter[str] = Counter()
|
| 318 |
+
engine.model.eval()
|
| 319 |
+
adapter.eval()
|
| 320 |
+
with torch.inference_mode():
|
| 321 |
+
for formula in samples:
|
| 322 |
+
groups = formula["truth_groups"]
|
| 323 |
+
boxes = [_group_box06(group, formula["strokes"]) for group in groups]
|
| 324 |
+
formula_box = _formula_box06(formula["strokes"])
|
| 325 |
+
sequences = [
|
| 326 |
+
_canonical_group06(group, formula["strokes"], formula_box)
|
| 327 |
+
for group in groups
|
| 328 |
+
]
|
| 329 |
+
logits_parts = []
|
| 330 |
+
for start in range(0, len(sequences), teacher_batch_size):
|
| 331 |
+
batch = torch.stack(sequences[start:start + teacher_batch_size]).to(device)
|
| 332 |
+
four_hypotheses = batch.unsqueeze(1).expand(-1, 4, -1, -1).contiguous()
|
| 333 |
+
logits, _family = _fused_exact_family_logits06(engine, adapter, four_hypotheses)
|
| 334 |
+
logits_parts.append(logits.cpu())
|
| 335 |
+
teacher_logits = torch.cat(logits_parts)
|
| 336 |
+
for index, label_value in enumerate(formula["truth_labels"]):
|
| 337 |
+
label = str(label_value)
|
| 338 |
+
if label not in TARGET_LABELS06:
|
| 339 |
+
continue
|
| 340 |
+
target = behavior_role_index06(label)
|
| 341 |
+
if target is None:
|
| 342 |
+
continue
|
| 343 |
+
sequence_rows.append(sequences[index])
|
| 344 |
+
context_rows.append(_context_row06(
|
| 345 |
+
formula, index, boxes, teacher_logits, labels, role_indices,
|
| 346 |
+
))
|
| 347 |
+
target_rows.append(target)
|
| 348 |
+
counts[BEHAVIOR_ROLE_LABELS06[target]] += 1
|
| 349 |
+
if return_metadata:
|
| 350 |
+
teacher_index = int(teacher_logits[index].argmax())
|
| 351 |
+
metadata_rows.append({
|
| 352 |
+
"sample_id": str(formula["sample_id"]),
|
| 353 |
+
"truth_label": label,
|
| 354 |
+
"teacher_label": labels[teacher_index],
|
| 355 |
+
"teacher_probability": float(
|
| 356 |
+
teacher_logits[index].softmax(dim=0)[teacher_index]
|
| 357 |
+
),
|
| 358 |
+
})
|
| 359 |
+
if not sequence_rows:
|
| 360 |
+
raise ValueError("행동 role 학습 표본이 없습니다.")
|
| 361 |
+
dataset = TensorDataset(
|
| 362 |
+
torch.stack(sequence_rows),
|
| 363 |
+
torch.tensor(context_rows, dtype=torch.float32),
|
| 364 |
+
torch.tensor(target_rows, dtype=torch.long),
|
| 365 |
+
)
|
| 366 |
+
if return_metadata:
|
| 367 |
+
return dataset, dict(counts), metadata_rows
|
| 368 |
+
return dataset, dict(counts)
|
| 369 |
+
|
| 370 |
+
|
| 371 |
+
def _synthetic_scale06(label: str, rng: np.random.Generator) -> float:
|
| 372 |
+
"""필요 변수: case label·seed RNG. 작동 원리: 실제 행의 겹치는 크기 분포를 포함한 수식 배치 높이를 만든다."""
|
| 373 |
+
|
| 374 |
+
ambiguous = bool(rng.random() < 0.20)
|
| 375 |
+
if label.isupper():
|
| 376 |
+
value = rng.normal(0.84 if ambiguous else 0.96, 0.055)
|
| 377 |
+
else:
|
| 378 |
+
value = rng.normal(0.84 if ambiguous else 0.68, 0.070)
|
| 379 |
+
return float(np.clip(value, 0.48, 1.05))
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def _place_synthetic_row06(feature: Tensor, scale: float) -> Tensor:
|
| 383 |
+
"""필요 변수: 실제 glyph 128×19 feature·합성 높이. 작동 원리: 필순은 보존하고 row-relative canvas/baseline만 배치한다."""
|
| 384 |
+
|
| 385 |
+
output = feature.clone()
|
| 386 |
+
top = (1.0 - scale) * 0.5
|
| 387 |
+
bottom = top + scale
|
| 388 |
+
output[:, 3] = top + output[:, 1] * scale
|
| 389 |
+
output[:, 10] = top
|
| 390 |
+
output[:, 11] = bottom
|
| 391 |
+
output[:, 12] = scale
|
| 392 |
+
output[:, 13] = 0.5
|
| 393 |
+
output[:, 14] = 1.0
|
| 394 |
+
return output
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
def _product_proxy_records06(args: argparse.Namespace, labels: Sequence[str]) -> list[dict[str, Any]]:
|
| 398 |
+
"""필요 변수: fail-closed registry·teacher vocabulary. 작동 원리: 승인 training split만 읽어 대소문자 proxy 원천을 만든다."""
|
| 399 |
+
|
| 400 |
+
selection_args = argparse.Namespace(
|
| 401 |
+
data=args.data,
|
| 402 |
+
commercial_paired=args.commercial_paired,
|
| 403 |
+
dataset_registry=args.dataset_registry,
|
| 404 |
+
source_registry=args.source_registry,
|
| 405 |
+
hwrt_approval=args.hwrt_approval,
|
| 406 |
+
paired_source=["uci-uji-pen-v1", "uci-uji-pen-v2"],
|
| 407 |
+
split="training",
|
| 408 |
+
maximum_paired_train_per_label=args.product_proxy_per_label,
|
| 409 |
+
maximum_paired_eval_per_label=0,
|
| 410 |
+
)
|
| 411 |
+
records, _fingerprint = _selected_records(selection_args, list(labels))
|
| 412 |
+
return records
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
def _materialize_product_proxy06(
|
| 416 |
+
records: Sequence[dict[str, Any]],
|
| 417 |
+
engine,
|
| 418 |
+
adapter: torch.nn.Module,
|
| 419 |
+
*,
|
| 420 |
+
seed: int,
|
| 421 |
+
device: torch.device,
|
| 422 |
+
teacher_batch_size: int,
|
| 423 |
+
target_bases: frozenset[str],
|
| 424 |
+
lowercase_ratio: float,
|
| 425 |
+
) -> tuple[TensorDataset, dict[str, Any]]:
|
| 426 |
+
"""필요 변수: 승인 고립 trajectory·frozen teacher. 작동 원리: 실제 stroke를 확률적 수식 행에 배치해 case 행동 proxy를 만든다."""
|
| 427 |
+
|
| 428 |
+
labels = tuple(str(label) for label in engine.labels)
|
| 429 |
+
role_indices = _teacher_role_indices06(labels)
|
| 430 |
+
rng = np.random.default_rng(seed + 600)
|
| 431 |
+
if not target_bases or not target_bases <= frozenset({"c", "o", "s", "u", "v", "w", "x", "z"}):
|
| 432 |
+
raise ValueError("제품 proxy target base가 지원 case family와 다릅니다.")
|
| 433 |
+
if not 0.0 <= lowercase_ratio <= 1.0:
|
| 434 |
+
raise ValueError("제품 proxy lowercase ratio는 0~1이어야 합니다.")
|
| 435 |
+
targets = [
|
| 436 |
+
record for record in records
|
| 437 |
+
if (
|
| 438 |
+
len(str(record["label"])) == 1
|
| 439 |
+
and str(record["label"]).lower() in target_bases
|
| 440 |
+
)
|
| 441 |
+
]
|
| 442 |
+
targets = [
|
| 443 |
+
record for index, record in enumerate(targets)
|
| 444 |
+
if (
|
| 445 |
+
str(record["label"]).isupper()
|
| 446 |
+
or (index * 2654435761 + seed) % 10_000 < int(lowercase_ratio * 10_000)
|
| 447 |
+
)
|
| 448 |
+
]
|
| 449 |
+
anchors = [record for record in records if str(record["label"]).isdigit()]
|
| 450 |
+
if not targets or not anchors:
|
| 451 |
+
raise ValueError("제품 proxy에 case target 또는 숫자 anchor가 없습니다.")
|
| 452 |
+
anchors_by_writer: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
| 453 |
+
anchors_by_source: dict[str, list[dict[str, Any]]] = {}
|
| 454 |
+
for record in anchors:
|
| 455 |
+
key = (str(record.get("source")), str(record.get("writer_key")))
|
| 456 |
+
anchors_by_writer.setdefault(key, []).append(record)
|
| 457 |
+
anchors_by_source.setdefault(str(record.get("source")), []).append(record)
|
| 458 |
+
sequences: list[Tensor] = []
|
| 459 |
+
triplet_features: list[Tensor] = []
|
| 460 |
+
metadata: list[dict[str, Any]] = []
|
| 461 |
+
for index, record in enumerate(targets):
|
| 462 |
+
key = (str(record.get("source")), str(record.get("writer_key")))
|
| 463 |
+
pool = anchors_by_writer.get(key) or anchors_by_source.get(str(record.get("source"))) or anchors
|
| 464 |
+
left_record = pool[(index * 2) % len(pool)]
|
| 465 |
+
right_record = pool[(index * 2 + 1) % len(pool)]
|
| 466 |
+
label = str(record["label"])
|
| 467 |
+
scale = _synthetic_scale06(label, rng)
|
| 468 |
+
target_feature = _place_synthetic_row06(_paired_record_feature06(record)[0], scale)
|
| 469 |
+
left_scale = float(np.clip(rng.normal(0.96, 0.04), 0.82, 1.05))
|
| 470 |
+
right_scale = float(np.clip(rng.normal(0.96, 0.04), 0.82, 1.05))
|
| 471 |
+
left_feature = _place_synthetic_row06(_paired_record_feature06(left_record)[0], left_scale)
|
| 472 |
+
right_feature = _place_synthetic_row06(_paired_record_feature06(right_record)[0], right_scale)
|
| 473 |
+
sequences.append(target_feature)
|
| 474 |
+
triplet_features.extend((target_feature, left_feature, right_feature))
|
| 475 |
+
metadata.append({
|
| 476 |
+
"record": record,
|
| 477 |
+
"label": label,
|
| 478 |
+
"scale": scale,
|
| 479 |
+
"left_scale": left_scale,
|
| 480 |
+
"right_scale": right_scale,
|
| 481 |
+
"gap_left": float(rng.uniform(0.12, 0.55)),
|
| 482 |
+
"gap_right": float(rng.uniform(0.12, 0.55)),
|
| 483 |
+
})
|
| 484 |
+
teacher_rows = []
|
| 485 |
+
engine.model.eval()
|
| 486 |
+
adapter.eval()
|
| 487 |
+
with torch.inference_mode():
|
| 488 |
+
for start in range(0, len(triplet_features), teacher_batch_size):
|
| 489 |
+
batch = torch.stack(triplet_features[start:start + teacher_batch_size]).to(device)
|
| 490 |
+
hypotheses = batch.unsqueeze(1).expand(-1, 4, -1, -1).contiguous()
|
| 491 |
+
logits, _family = _fused_exact_family_logits06(engine, adapter, hypotheses)
|
| 492 |
+
teacher_rows.append(logits.cpu())
|
| 493 |
+
teacher_logits = torch.cat(teacher_rows).reshape(len(targets), 3, -1)
|
| 494 |
+
context_rows: list[list[float]] = []
|
| 495 |
+
target_rows: list[int] = []
|
| 496 |
+
counts: Counter[str] = Counter()
|
| 497 |
+
ambiguous_rows = 0
|
| 498 |
+
for index, row in enumerate(metadata):
|
| 499 |
+
label = row["label"]
|
| 500 |
+
target = behavior_role_index06(label)
|
| 501 |
+
if target is None:
|
| 502 |
+
continue
|
| 503 |
+
probability = teacher_logits[index].softmax(dim=1)
|
| 504 |
+
log_probability = teacher_logits[index, 0].log_softmax(dim=0)
|
| 505 |
+
base = label.lower()
|
| 506 |
+
lower, upper, times, family_mass = _family_log_probability06(
|
| 507 |
+
log_probability, labels, base,
|
| 508 |
+
)
|
| 509 |
+
target_probability = probability[0]
|
| 510 |
+
entropy = float(
|
| 511 |
+
-(target_probability * target_probability.clamp_min(1e-9).log()).sum()
|
| 512 |
+
/ max(math.log(len(labels)), 1.0)
|
| 513 |
+
)
|
| 514 |
+
left_role = _role_mass06(probability[1], role_indices)
|
| 515 |
+
right_role = _role_mass06(probability[2], role_indices)
|
| 516 |
+
record = row["record"]
|
| 517 |
+
target_strokes = list(record["strokes"])
|
| 518 |
+
cross = np.zeros(len(CROSS_FEATURE_NAMES), dtype=np.float32)
|
| 519 |
+
if len(target_strokes) == 2:
|
| 520 |
+
cross = cross_pair_feature_rows([(0, 1)], target_strokes)[0]
|
| 521 |
+
point_count = sum(len(stroke.get("points") or []) for stroke in target_strokes)
|
| 522 |
+
scale = float(row["scale"])
|
| 523 |
+
reference_height = (float(row["left_scale"]) + float(row["right_scale"])) * 0.5
|
| 524 |
+
aspect = float(sequences[index][0, 9])
|
| 525 |
+
values = [
|
| 526 |
+
lower, upper, times, family_mass,
|
| 527 |
+
float(target_probability.max()), entropy,
|
| 528 |
+
*left_role, *right_role,
|
| 529 |
+
float(row["gap_left"]), float(row["gap_right"]), 0.0, 0.0,
|
| 530 |
+
0.5, 0.5, min(0.24 * max(aspect, 0.2), 0.45), scale,
|
| 531 |
+
scale / max(reference_height, 1e-5),
|
| 532 |
+
max(aspect, 1e-3),
|
| 533 |
+
min(len(target_strokes) / 8.0, 1.0),
|
| 534 |
+
min(point_count / 256.0, 1.0),
|
| 535 |
+
float(base == "c"), float(base == "x"), float(base == "z"),
|
| 536 |
+
*cross.tolist(),
|
| 537 |
+
float(len(target_strokes) == 2),
|
| 538 |
+
]
|
| 539 |
+
validate_behavior_context06(values)
|
| 540 |
+
context_rows.append(values)
|
| 541 |
+
target_rows.append(target)
|
| 542 |
+
counts[BEHAVIOR_ROLE_LABELS06[target]] += 1
|
| 543 |
+
ambiguous_rows += int(
|
| 544 |
+
(label.isupper() and scale < 0.90)
|
| 545 |
+
or (label.islower() and scale > 0.78)
|
| 546 |
+
)
|
| 547 |
+
return TensorDataset(
|
| 548 |
+
torch.stack(sequences),
|
| 549 |
+
torch.tensor(context_rows, dtype=torch.float32),
|
| 550 |
+
torch.tensor(target_rows, dtype=torch.long),
|
| 551 |
+
), {
|
| 552 |
+
"samples": len(target_rows),
|
| 553 |
+
"role_counts": dict(counts),
|
| 554 |
+
"ambiguous_size_samples": ambiguous_rows,
|
| 555 |
+
"sources": dict(Counter(str(record.get("source")) for record in targets)),
|
| 556 |
+
"track": "P_approved_synthetic_layout_proxy",
|
| 557 |
+
"actual_continuous_formula": False,
|
| 558 |
+
"target_bases": sorted(target_bases),
|
| 559 |
+
"lowercase_ratio": lowercase_ratio,
|
| 560 |
+
}
|
| 561 |
+
|
| 562 |
+
|
| 563 |
+
def _with_weight06(dataset: TensorDataset, weight: float) -> TensorDataset:
|
| 564 |
+
"""필요 변수: dataset·loss weight. 작동 원리: source별 기여도를 각 행에 명시적으로 붙인다."""
|
| 565 |
+
|
| 566 |
+
return TensorDataset(
|
| 567 |
+
*dataset.tensors,
|
| 568 |
+
torch.full((len(dataset),), float(weight), dtype=torch.float32),
|
| 569 |
+
)
|
| 570 |
+
|
| 571 |
+
|
| 572 |
+
def _combine_training06(parts: Sequence[TensorDataset]) -> TensorDataset:
|
| 573 |
+
"""필요 변수: 동일 계약의 weighted dataset. 작동 원리: source 순서를 보존해 한 학습 tensor로 결합한다."""
|
| 574 |
+
|
| 575 |
+
return TensorDataset(*(
|
| 576 |
+
torch.cat([dataset.tensors[index] for dataset in parts], dim=0)
|
| 577 |
+
for index in range(len(parts[0].tensors))
|
| 578 |
+
))
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
def _normalize_context06(
|
| 582 |
+
training: TensorDataset,
|
| 583 |
+
validation: TensorDataset,
|
| 584 |
+
testing: TensorDataset,
|
| 585 |
+
*,
|
| 586 |
+
statistics_reference: TensorDataset | None = None,
|
| 587 |
+
) -> tuple[TensorDataset, TensorDataset, TensorDataset, Tensor, Tensor]:
|
| 588 |
+
"""필요 변수: 세 split context. 작동 원리: fit 통계만 사용해 validation/test 누수를 차단한다."""
|
| 589 |
+
|
| 590 |
+
reference = statistics_reference or training
|
| 591 |
+
mean = reference.tensors[1].mean(dim=0)
|
| 592 |
+
scale = reference.tensors[1].std(dim=0).clamp_min(1e-5)
|
| 593 |
+
|
| 594 |
+
def normalized(dataset: TensorDataset) -> TensorDataset:
|
| 595 |
+
"""필요 변수: 원본 dataset. 작동 원리: sequence/target은 보존하고 context만 표준화한다."""
|
| 596 |
+
|
| 597 |
+
return TensorDataset(
|
| 598 |
+
dataset.tensors[0],
|
| 599 |
+
(dataset.tensors[1] - mean) / scale,
|
| 600 |
+
*dataset.tensors[2:],
|
| 601 |
+
)
|
| 602 |
+
|
| 603 |
+
return normalized(training), normalized(validation), normalized(testing), mean, scale
|
| 604 |
+
|
| 605 |
+
|
| 606 |
+
def _metrics06(logits: Tensor, target: Tensor) -> dict[str, Any]:
|
| 607 |
+
"""필요 변수: N×3 logits·정답. 작동 원리: accuracy·macro-F1·class recall·ECE를 같은 분모로 계산한다."""
|
| 608 |
+
|
| 609 |
+
prediction = logits.argmax(dim=1)
|
| 610 |
+
probability = logits.softmax(dim=1)
|
| 611 |
+
confusion = torch.zeros((3, 3), dtype=torch.long)
|
| 612 |
+
for truth, selected in zip(target.tolist(), prediction.tolist(), strict=True):
|
| 613 |
+
confusion[truth, selected] += 1
|
| 614 |
+
f1_values, recalls = [], {}
|
| 615 |
+
for index, label in enumerate(BEHAVIOR_ROLE_LABELS06):
|
| 616 |
+
true_positive = int(confusion[index, index])
|
| 617 |
+
false_positive = int(confusion[:, index].sum()) - true_positive
|
| 618 |
+
false_negative = int(confusion[index, :].sum()) - true_positive
|
| 619 |
+
precision = true_positive / max(true_positive + false_positive, 1)
|
| 620 |
+
recall = true_positive / max(true_positive + false_negative, 1)
|
| 621 |
+
f1_values.append(2 * precision * recall / max(precision + recall, 1e-12))
|
| 622 |
+
recalls[label] = recall
|
| 623 |
+
confidence = probability.max(dim=1).values
|
| 624 |
+
correct = prediction.eq(target)
|
| 625 |
+
ece = 0.0
|
| 626 |
+
for lower in torch.arange(0.0, 1.0, 0.1):
|
| 627 |
+
selected = (confidence >= lower) & (confidence < lower + 0.1)
|
| 628 |
+
if selected.any():
|
| 629 |
+
ece += float(selected.float().mean()) * abs(
|
| 630 |
+
float(correct[selected].float().mean()) - float(confidence[selected].mean())
|
| 631 |
+
)
|
| 632 |
+
return {
|
| 633 |
+
"samples": len(target),
|
| 634 |
+
"accuracy": float(correct.float().mean()),
|
| 635 |
+
"macro_f1": float(np.mean(f1_values)),
|
| 636 |
+
"recall": recalls,
|
| 637 |
+
"confusion": confusion.tolist(),
|
| 638 |
+
"ece": ece,
|
| 639 |
+
}
|
| 640 |
+
|
| 641 |
+
|
| 642 |
+
def _evaluate06(
|
| 643 |
+
model: BehaviorRoleHead06,
|
| 644 |
+
dataset: TensorDataset,
|
| 645 |
+
*,
|
| 646 |
+
batch_size: int,
|
| 647 |
+
device: torch.device,
|
| 648 |
+
) -> tuple[dict[str, Any], Tensor, Tensor]:
|
| 649 |
+
"""필요 변수: behavior head·split. 작동 원리: 순서를 보존해 전체 logits와 정답을 모은다."""
|
| 650 |
+
|
| 651 |
+
model.eval()
|
| 652 |
+
logits_rows, target_rows = [], []
|
| 653 |
+
with torch.inference_mode():
|
| 654 |
+
for batch in DataLoader(dataset, batch_size=batch_size, shuffle=False):
|
| 655 |
+
sequence, context, target = batch[:3]
|
| 656 |
+
logits_rows.append(model(sequence.to(device), context.to(device)).cpu())
|
| 657 |
+
target_rows.append(target)
|
| 658 |
+
logits, targets = torch.cat(logits_rows), torch.cat(target_rows)
|
| 659 |
+
return _metrics06(logits, targets), logits, targets
|
| 660 |
+
|
| 661 |
+
|
| 662 |
+
def _teacher_baseline06(dataset: TensorDataset) -> dict[str, Any]:
|
| 663 |
+
"""필요 변수: 정규화 전 context의 teacher logp 3개. 작동 원리: 행동 head가 실제로 teacher family를 개선했는지 비교한다."""
|
| 664 |
+
|
| 665 |
+
logits = dataset.tensors[1][:, :3]
|
| 666 |
+
return _metrics06(logits, dataset.tensors[2])
|
| 667 |
+
|
| 668 |
+
|
| 669 |
+
def main() -> None:
|
| 670 |
+
"""필요 변수: seed별 adapter와 CROHME 공식 split. 작동 원리: validation 선택 후 official test를 한 번 평가한다."""
|
| 671 |
+
|
| 672 |
+
args = _parse_args()
|
| 673 |
+
if args.train_root.name.casefold() != "traindata" or args.test_root.name.casefold() != "testdatagt":
|
| 674 |
+
raise ValueError("CROHME2012 공식 trainData/testDataGT 조합만 허용합니다.")
|
| 675 |
+
device = torch.device(args.device)
|
| 676 |
+
if device.type == "cuda" and not torch.cuda.is_available():
|
| 677 |
+
raise RuntimeError("CUDA 학습을 요청했지만 사용할 수 없습니다.")
|
| 678 |
+
_seed06(args.seed)
|
| 679 |
+
adapter_payload = torch.load(args.adapter, map_location="cpu", weights_only=False)
|
| 680 |
+
base_checkpoint = Path(str(adapter_payload["base_checkpoint"]))
|
| 681 |
+
if not base_checkpoint.is_absolute():
|
| 682 |
+
base_checkpoint = PROJECT_ROOT / base_checkpoint
|
| 683 |
+
engine, adapter = _load_model06(base_checkpoint, args.adapter, device)
|
| 684 |
+
fit_samples, validation_samples = writer_fit_validation(args.train_root, args.profile)
|
| 685 |
+
test_samples = _samples(args.test_root, args.profile)
|
| 686 |
+
training_raw, training_counts = _materialize_split06(
|
| 687 |
+
fit_samples, engine, adapter, device=device, teacher_batch_size=args.teacher_batch_size,
|
| 688 |
+
)
|
| 689 |
+
validation_raw, validation_counts = _materialize_split06(
|
| 690 |
+
validation_samples, engine, adapter, device=device, teacher_batch_size=args.teacher_batch_size,
|
| 691 |
+
)
|
| 692 |
+
testing_raw, testing_counts = _materialize_split06(
|
| 693 |
+
test_samples, engine, adapter, device=device, teacher_batch_size=args.teacher_batch_size,
|
| 694 |
+
)
|
| 695 |
+
proxy_info: dict[str, Any] | None = None
|
| 696 |
+
training_weighted = _with_weight06(training_raw, 1.0)
|
| 697 |
+
if args.product_proxy_per_label > 0:
|
| 698 |
+
product_records = _product_proxy_records06(args, engine.labels)
|
| 699 |
+
proxy_raw, proxy_info = _materialize_product_proxy06(
|
| 700 |
+
product_records,
|
| 701 |
+
engine,
|
| 702 |
+
adapter,
|
| 703 |
+
seed=args.seed,
|
| 704 |
+
device=device,
|
| 705 |
+
teacher_batch_size=args.teacher_batch_size,
|
| 706 |
+
target_bases=frozenset(args.product_proxy_target_bases),
|
| 707 |
+
lowercase_ratio=args.product_proxy_lowercase_ratio,
|
| 708 |
+
)
|
| 709 |
+
training_weighted = _combine_training06((
|
| 710 |
+
training_weighted,
|
| 711 |
+
_with_weight06(proxy_raw, args.product_proxy_loss_weight),
|
| 712 |
+
))
|
| 713 |
+
del engine, adapter
|
| 714 |
+
if device.type == "cuda":
|
| 715 |
+
torch.cuda.empty_cache()
|
| 716 |
+
teacher_baseline = {
|
| 717 |
+
"validation": _teacher_baseline06(validation_raw),
|
| 718 |
+
"official_test": _teacher_baseline06(testing_raw),
|
| 719 |
+
}
|
| 720 |
+
training, validation, testing, mean, scale = _normalize_context06(
|
| 721 |
+
training_weighted, validation_raw, testing_raw,
|
| 722 |
+
statistics_reference=training_raw,
|
| 723 |
+
)
|
| 724 |
+
model = BehaviorRoleHead06(hidden=args.hidden, dropout=args.dropout).to(device)
|
| 725 |
+
counts = torch.bincount(training.tensors[2], minlength=3).float()
|
| 726 |
+
class_weight = (counts.sum() / counts.clamp_min(1.0)).sqrt()
|
| 727 |
+
class_weight = (class_weight / class_weight.mean()).to(device)
|
| 728 |
+
optimizer = torch.optim.AdamW(
|
| 729 |
+
model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay,
|
| 730 |
+
)
|
| 731 |
+
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.epochs)
|
| 732 |
+
loader = DataLoader(
|
| 733 |
+
training, batch_size=args.batch_size, shuffle=True,
|
| 734 |
+
generator=torch.Generator().manual_seed(args.seed),
|
| 735 |
+
)
|
| 736 |
+
best_key = (-1.0, -1.0)
|
| 737 |
+
best_state: dict[str, Tensor] | None = None
|
| 738 |
+
best_epoch = 0
|
| 739 |
+
history = []
|
| 740 |
+
stale = 0
|
| 741 |
+
for epoch in range(1, args.epochs + 1):
|
| 742 |
+
model.train()
|
| 743 |
+
total_loss = 0.0
|
| 744 |
+
samples = 0
|
| 745 |
+
for sequence, context, target, sample_weight in loader:
|
| 746 |
+
sequence, context = sequence.to(device), context.to(device)
|
| 747 |
+
target, sample_weight = target.to(device), sample_weight.to(device)
|
| 748 |
+
optimizer.zero_grad(set_to_none=True)
|
| 749 |
+
logits = model(sequence, context)
|
| 750 |
+
losses = torch.nn.functional.cross_entropy(
|
| 751 |
+
logits, target, weight=class_weight, reduction="none",
|
| 752 |
+
)
|
| 753 |
+
loss = (losses * sample_weight).sum() / sample_weight.sum().clamp_min(1e-8)
|
| 754 |
+
loss.backward()
|
| 755 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 2.0)
|
| 756 |
+
optimizer.step()
|
| 757 |
+
total_loss += float(loss.detach()) * len(target)
|
| 758 |
+
samples += len(target)
|
| 759 |
+
scheduler.step()
|
| 760 |
+
validation_metrics, _logits, _targets = _evaluate06(
|
| 761 |
+
model, validation, batch_size=args.batch_size, device=device,
|
| 762 |
+
)
|
| 763 |
+
row = {
|
| 764 |
+
"epoch": epoch,
|
| 765 |
+
"training_loss": total_loss / max(samples, 1),
|
| 766 |
+
"learning_rate": optimizer.param_groups[0]["lr"],
|
| 767 |
+
"validation": validation_metrics,
|
| 768 |
+
}
|
| 769 |
+
history.append(row)
|
| 770 |
+
key = (validation_metrics["macro_f1"], validation_metrics["accuracy"])
|
| 771 |
+
if key > best_key:
|
| 772 |
+
best_key = key
|
| 773 |
+
best_epoch = epoch
|
| 774 |
+
best_state = deepcopy({key: value.detach().cpu() for key, value in model.state_dict().items()})
|
| 775 |
+
stale = 0
|
| 776 |
+
else:
|
| 777 |
+
stale += 1
|
| 778 |
+
if epoch == 1 or epoch % 5 == 0:
|
| 779 |
+
print(json.dumps({
|
| 780 |
+
"seed": args.seed, "epoch": epoch, "loss": row["training_loss"],
|
| 781 |
+
"validation_accuracy": validation_metrics["accuracy"],
|
| 782 |
+
"validation_macro_f1": validation_metrics["macro_f1"],
|
| 783 |
+
}, ensure_ascii=False), flush=True)
|
| 784 |
+
if stale >= args.patience:
|
| 785 |
+
break
|
| 786 |
+
if best_state is None:
|
| 787 |
+
raise RuntimeError("behavior role checkpoint가 선택되지 않았습니다.")
|
| 788 |
+
model.load_state_dict(best_state)
|
| 789 |
+
validation_metrics, _validation_logits, _validation_targets = _evaluate06(
|
| 790 |
+
model, validation, batch_size=args.batch_size, device=device,
|
| 791 |
+
)
|
| 792 |
+
test_metrics, _test_logits, _test_targets = _evaluate06(
|
| 793 |
+
model, testing, batch_size=args.batch_size, device=device,
|
| 794 |
+
)
|
| 795 |
+
args.output.mkdir(parents=True, exist_ok=True)
|
| 796 |
+
checkpoint = args.output / "behavior_role_head.pt"
|
| 797 |
+
torch.save({
|
| 798 |
+
"schema": "aiflow-math-ink-06-behavior-role-v1",
|
| 799 |
+
"state_dict": best_state,
|
| 800 |
+
"context_mean": mean,
|
| 801 |
+
"context_scale": scale,
|
| 802 |
+
"context_features": BEHAVIOR_CONTEXT_FEATURES06,
|
| 803 |
+
"role_labels": BEHAVIOR_ROLE_LABELS06,
|
| 804 |
+
"sequence_channels": 19,
|
| 805 |
+
"hidden": args.hidden,
|
| 806 |
+
"dropout": args.dropout,
|
| 807 |
+
"seed": args.seed,
|
| 808 |
+
"teacher_adapter": str(args.adapter),
|
| 809 |
+
"selected_epoch": best_epoch,
|
| 810 |
+
"track": "R_noncommercial_only",
|
| 811 |
+
"product_validation": False,
|
| 812 |
+
}, checkpoint)
|
| 813 |
+
report = {
|
| 814 |
+
"experiment": "R-MATH-INK-06-BEHAVIOR-ROLE-001",
|
| 815 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 816 |
+
"seed": args.seed,
|
| 817 |
+
"device": str(device),
|
| 818 |
+
"cuda_device": torch.cuda.get_device_name(0) if device.type == "cuda" else None,
|
| 819 |
+
"teacher_adapter": str(args.adapter),
|
| 820 |
+
"base_checkpoint": str(base_checkpoint),
|
| 821 |
+
"split_contract": "CROHME2012 trainData writer fit/validation; testDataGT official held-out",
|
| 822 |
+
"formulas": {
|
| 823 |
+
"fit": len(fit_samples), "validation": len(validation_samples), "official_test": len(test_samples),
|
| 824 |
+
},
|
| 825 |
+
"role_samples": {
|
| 826 |
+
"fit": training_counts, "validation": validation_counts, "official_test": testing_counts,
|
| 827 |
+
},
|
| 828 |
+
"product_proxy": proxy_info,
|
| 829 |
+
"product_proxy_loss_weight": (
|
| 830 |
+
args.product_proxy_loss_weight if proxy_info is not None else 0.0
|
| 831 |
+
),
|
| 832 |
+
"teacher_baseline": teacher_baseline,
|
| 833 |
+
"selected_epoch": best_epoch,
|
| 834 |
+
"validation": validation_metrics,
|
| 835 |
+
"official_test": test_metrics,
|
| 836 |
+
"history": history,
|
| 837 |
+
"checkpoint": checkpoint.name,
|
| 838 |
+
"checkpoint_bytes": checkpoint.stat().st_size,
|
| 839 |
+
"track": "R_noncommercial_only",
|
| 840 |
+
"product_validation": False,
|
| 841 |
+
"interpretation_limit": (
|
| 842 |
+
"CROHME 연구용 연속 수식 행동 head이며 상용 checkpoint에 병합할 수 없다. "
|
| 843 |
+
"P-track writer/device-disjoint 재학습이 필요하다."
|
| 844 |
+
),
|
| 845 |
+
}
|
| 846 |
+
(args.output / "report.json").write_text(
|
| 847 |
+
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
|
| 848 |
+
encoding="utf-8",
|
| 849 |
+
)
|
| 850 |
+
print(json.dumps({
|
| 851 |
+
"seed": args.seed,
|
| 852 |
+
"selected_epoch": best_epoch,
|
| 853 |
+
"teacher_test": teacher_baseline["official_test"],
|
| 854 |
+
"behavior_test": test_metrics,
|
| 855 |
+
"checkpoint": str(checkpoint),
|
| 856 |
+
"product_validation": False,
|
| 857 |
+
}, ensure_ascii=False, indent=2))
|
| 858 |
+
|
| 859 |
+
|
| 860 |
+
if __name__ == "__main__":
|
| 861 |
+
main()
|
scripts/train_math_ink_06_formula_adapter.py
ADDED
|
@@ -0,0 +1,373 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CROHME truth group에서 연속식 domain residual adapter의 구조적 회복 가능성을 진단한다."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from collections import Counter
|
| 7 |
+
from copy import deepcopy
|
| 8 |
+
from datetime import datetime, timezone
|
| 9 |
+
import json
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
import random
|
| 12 |
+
import sys
|
| 13 |
+
from typing import Sequence
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
import torch
|
| 17 |
+
from torch import Tensor
|
| 18 |
+
from torch.utils.data import DataLoader, TensorDataset, WeightedRandomSampler
|
| 19 |
+
|
| 20 |
+
PROJECT_ROOT = Path(__file__).parents[1]
|
| 21 |
+
SOURCE_ROOT = PROJECT_ROOT / "src"
|
| 22 |
+
for path in (PROJECT_ROOT, SOURCE_ROOT):
|
| 23 |
+
if str(path) not in sys.path:
|
| 24 |
+
sys.path.insert(0, str(path))
|
| 25 |
+
|
| 26 |
+
from math_grid_drawer.research.skeleton_adapter06 import SkeletonTrajectoryAdapter06
|
| 27 |
+
from math_grid_drawer.research.trajectory_sequence import shape_family, visual_label_family
|
| 28 |
+
from scripts.audit_math_ink_06_case_context import _load_model06
|
| 29 |
+
from scripts.audit_math_ink_06_formula_context_contract import _materialize_formula_groups06
|
| 30 |
+
from scripts.crohme_lattice_common import writer_fit_validation
|
| 31 |
+
from scripts.train_crohme_segmentation_lattice_selector import _samples
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _parse_args() -> argparse.Namespace:
|
| 35 |
+
"""필요 변수: 제품 encoder adapter·공식 CROHME split·학습 설정. 작동 원리: R-track shadow adapter CLI를 만든다."""
|
| 36 |
+
|
| 37 |
+
parser = argparse.ArgumentParser(description="Train Math Ink 0.6 formula domain adapter")
|
| 38 |
+
parser.add_argument("--adapter", type=Path, required=True)
|
| 39 |
+
parser.add_argument(
|
| 40 |
+
"--train-root", type=Path,
|
| 41 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/trainData",
|
| 42 |
+
)
|
| 43 |
+
parser.add_argument(
|
| 44 |
+
"--test-root", type=Path,
|
| 45 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/testDataGT",
|
| 46 |
+
)
|
| 47 |
+
parser.add_argument("--profile", default="median_height_32")
|
| 48 |
+
parser.add_argument("--seed", type=int, default=17)
|
| 49 |
+
parser.add_argument("--epochs", type=int, default=12)
|
| 50 |
+
parser.add_argument("--batch-size", type=int, default=256)
|
| 51 |
+
parser.add_argument("--learning-rate", type=float, default=4e-4)
|
| 52 |
+
parser.add_argument("--weight-decay", type=float, default=2e-3)
|
| 53 |
+
parser.add_argument("--exact-loss-weight", type=float, default=0.20)
|
| 54 |
+
parser.add_argument("--context-dropout", type=float, default=0.20)
|
| 55 |
+
parser.add_argument("--hidden-size", type=int, default=48)
|
| 56 |
+
parser.add_argument("--patience", type=int, default=4)
|
| 57 |
+
parser.add_argument("--skip-official-test", action="store_true")
|
| 58 |
+
parser.add_argument("--device", choices=("cuda", "cpu"), default="cuda")
|
| 59 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 60 |
+
return parser.parse_args()
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _seed06(seed: int) -> None:
|
| 64 |
+
"""필요 변수: seed. 작동 원리: Python·NumPy·PyTorch 난수를 함께 고정한다."""
|
| 65 |
+
|
| 66 |
+
random.seed(seed)
|
| 67 |
+
np.random.seed(seed)
|
| 68 |
+
torch.manual_seed(seed)
|
| 69 |
+
if torch.cuda.is_available():
|
| 70 |
+
torch.cuda.manual_seed_all(seed)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _targets06(
|
| 74 |
+
truths: Sequence[str],
|
| 75 |
+
exact_labels: Sequence[str],
|
| 76 |
+
family_labels: Sequence[str],
|
| 77 |
+
) -> tuple[Tensor, Tensor]:
|
| 78 |
+
"""필요 변수: truth label·model ontology. 작동 원리: exact/family target index를 만든다."""
|
| 79 |
+
|
| 80 |
+
exact_index = {str(label): index for index, label in enumerate(exact_labels)}
|
| 81 |
+
family_index = {str(label): index for index, label in enumerate(family_labels)}
|
| 82 |
+
return (
|
| 83 |
+
torch.tensor([exact_index[str(label)] for label in truths], dtype=torch.long),
|
| 84 |
+
torch.tensor([
|
| 85 |
+
family_index[shape_family(str(label))] for label in truths
|
| 86 |
+
], dtype=torch.long),
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _metrics06(
|
| 91 |
+
exact_logits: Tensor,
|
| 92 |
+
family_logits: Tensor,
|
| 93 |
+
exact_targets: Tensor,
|
| 94 |
+
family_targets: Tensor,
|
| 95 |
+
exact_labels: Sequence[str],
|
| 96 |
+
) -> dict[str, float | int]:
|
| 97 |
+
"""필요 변수: exact/family logit·target. 작동 원리: exact·shape-family·visual-family 지표를 같은 분모에서 계산한다."""
|
| 98 |
+
|
| 99 |
+
exact_prediction = exact_logits.argmax(dim=1)
|
| 100 |
+
family_prediction = family_logits.argmax(dim=1)
|
| 101 |
+
visual_hits = sum(
|
| 102 |
+
visual_label_family(str(exact_labels[int(prediction)]))
|
| 103 |
+
== visual_label_family(str(exact_labels[int(truth)]))
|
| 104 |
+
for prediction, truth in zip(
|
| 105 |
+
exact_prediction.tolist(), exact_targets.tolist(), strict=True,
|
| 106 |
+
)
|
| 107 |
+
)
|
| 108 |
+
return {
|
| 109 |
+
"samples": len(exact_targets),
|
| 110 |
+
"exact_top1": float(exact_prediction.eq(exact_targets).float().mean()),
|
| 111 |
+
"family_head_top1": float(family_prediction.eq(family_targets).float().mean()),
|
| 112 |
+
"visual_family_top1": visual_hits / max(len(exact_targets), 1),
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def _forward06(
|
| 117 |
+
model,
|
| 118 |
+
online_adapter,
|
| 119 |
+
formula_adapter,
|
| 120 |
+
features: Tensor,
|
| 121 |
+
*,
|
| 122 |
+
device: torch.device,
|
| 123 |
+
batch_size: int,
|
| 124 |
+
) -> tuple[Tensor, Tensor]:
|
| 125 |
+
"""필요 변수: frozen main·formula adapter·feature. 작동 원리: 전체 split logit을 순서대로 CPU에 반환한다."""
|
| 126 |
+
|
| 127 |
+
exact_rows, family_rows = [], []
|
| 128 |
+
model.eval()
|
| 129 |
+
online_adapter.eval()
|
| 130 |
+
formula_adapter.eval()
|
| 131 |
+
with torch.inference_mode():
|
| 132 |
+
for start in range(0, len(features), batch_size):
|
| 133 |
+
batch = features[start:start + batch_size].to(device)
|
| 134 |
+
exact, family = model.classify_trajectory(
|
| 135 |
+
formula_adapter(online_adapter(batch)),
|
| 136 |
+
)
|
| 137 |
+
exact_rows.append(exact.cpu())
|
| 138 |
+
family_rows.append(family.cpu())
|
| 139 |
+
return torch.cat(exact_rows), torch.cat(family_rows)
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def _weighted_loader06(
|
| 143 |
+
dataset: TensorDataset,
|
| 144 |
+
exact_targets: Tensor,
|
| 145 |
+
*,
|
| 146 |
+
batch_size: int,
|
| 147 |
+
seed: int,
|
| 148 |
+
) -> DataLoader:
|
| 149 |
+
"""필요 변수: 학습 tensor·exact target. 작동 원리: label 빈도 역제곱근 sampler로 대형 class 독점을 완화한다."""
|
| 150 |
+
|
| 151 |
+
counts = torch.bincount(exact_targets).float()
|
| 152 |
+
class_weight = counts.clamp_min(1.0).rsqrt()
|
| 153 |
+
weights = class_weight[exact_targets]
|
| 154 |
+
sampler = WeightedRandomSampler(
|
| 155 |
+
weights, num_samples=len(dataset), replacement=True,
|
| 156 |
+
generator=torch.Generator().manual_seed(seed),
|
| 157 |
+
)
|
| 158 |
+
return DataLoader(dataset, batch_size=batch_size, sampler=sampler)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def main() -> None:
|
| 162 |
+
"""필요 변수: writer fit/validation·official test. 작동 원리: frozen product encoder 앞 residual만 학습해 구조 상한을 측정한다."""
|
| 163 |
+
|
| 164 |
+
args = _parse_args()
|
| 165 |
+
if args.train_root.name.casefold() != "traindata" or args.test_root.name.casefold() != "testdatagt":
|
| 166 |
+
raise ValueError("CROHME2012 공식 trainData/testDataGT 조합만 허용합니다.")
|
| 167 |
+
if not 0.0 <= args.context_dropout <= 1.0:
|
| 168 |
+
raise ValueError("context dropout은 0~1 범위여야 합니다.")
|
| 169 |
+
device = torch.device(args.device)
|
| 170 |
+
if device.type == "cuda" and not torch.cuda.is_available():
|
| 171 |
+
raise RuntimeError("CUDA 학습을 요청했지만 사용할 수 없습니다.")
|
| 172 |
+
_seed06(args.seed)
|
| 173 |
+
adapter_payload = torch.load(args.adapter, map_location="cpu", weights_only=False)
|
| 174 |
+
base_checkpoint = Path(str(adapter_payload["base_checkpoint"]))
|
| 175 |
+
if not base_checkpoint.is_absolute():
|
| 176 |
+
base_checkpoint = PROJECT_ROOT / base_checkpoint
|
| 177 |
+
engine, online_adapter = _load_model06(base_checkpoint, args.adapter, device)
|
| 178 |
+
for parameter in engine.model.parameters():
|
| 179 |
+
parameter.requires_grad_(False)
|
| 180 |
+
for parameter in online_adapter.parameters():
|
| 181 |
+
parameter.requires_grad_(False)
|
| 182 |
+
fit_samples, validation_samples = writer_fit_validation(args.train_root, args.profile)
|
| 183 |
+
test_samples = [] if args.skip_official_test else _samples(args.test_root, args.profile)
|
| 184 |
+
train_x, train_truths, _train_behavior = _materialize_formula_groups06(
|
| 185 |
+
fit_samples, engine.labels,
|
| 186 |
+
)
|
| 187 |
+
validation_x, validation_truths, _validation_behavior = _materialize_formula_groups06(
|
| 188 |
+
validation_samples, engine.labels,
|
| 189 |
+
)
|
| 190 |
+
if test_samples:
|
| 191 |
+
test_x, test_truths, _test_behavior = _materialize_formula_groups06(
|
| 192 |
+
test_samples, engine.labels,
|
| 193 |
+
)
|
| 194 |
+
else:
|
| 195 |
+
test_x = torch.empty((0, 128, 19), dtype=train_x.dtype)
|
| 196 |
+
test_truths = []
|
| 197 |
+
train_exact, train_family = _targets06(
|
| 198 |
+
train_truths, engine.labels, engine.family_labels,
|
| 199 |
+
)
|
| 200 |
+
validation_exact, validation_family = _targets06(
|
| 201 |
+
validation_truths, engine.labels, engine.family_labels,
|
| 202 |
+
)
|
| 203 |
+
test_exact, test_family = (
|
| 204 |
+
_targets06(test_truths, engine.labels, engine.family_labels)
|
| 205 |
+
if test_truths else (
|
| 206 |
+
torch.empty(0, dtype=torch.long),
|
| 207 |
+
torch.empty(0, dtype=torch.long),
|
| 208 |
+
)
|
| 209 |
+
)
|
| 210 |
+
training = TensorDataset(train_x, train_exact, train_family)
|
| 211 |
+
loader = _weighted_loader06(
|
| 212 |
+
training, train_exact, batch_size=args.batch_size, seed=args.seed,
|
| 213 |
+
)
|
| 214 |
+
formula_adapter = SkeletonTrajectoryAdapter06(hidden_size=args.hidden_size).to(device)
|
| 215 |
+
optimizer = torch.optim.AdamW(
|
| 216 |
+
formula_adapter.parameters(), lr=args.learning_rate,
|
| 217 |
+
weight_decay=args.weight_decay,
|
| 218 |
+
)
|
| 219 |
+
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
|
| 220 |
+
optimizer, T_max=max(args.epochs, 1), eta_min=args.learning_rate * 0.1,
|
| 221 |
+
)
|
| 222 |
+
baseline_exact, baseline_family = _forward06(
|
| 223 |
+
engine.model, online_adapter, torch.nn.Identity().to(device), validation_x,
|
| 224 |
+
device=device, batch_size=args.batch_size,
|
| 225 |
+
)
|
| 226 |
+
baseline_validation = _metrics06(
|
| 227 |
+
baseline_exact, baseline_family, validation_exact, validation_family,
|
| 228 |
+
engine.labels,
|
| 229 |
+
)
|
| 230 |
+
best_key = (-1.0, -1.0)
|
| 231 |
+
best_state = None
|
| 232 |
+
best_epoch = 0
|
| 233 |
+
stale = 0
|
| 234 |
+
history = []
|
| 235 |
+
for epoch in range(1, args.epochs + 1):
|
| 236 |
+
formula_adapter.train()
|
| 237 |
+
losses = []
|
| 238 |
+
for features, exact_target, family_target in loader:
|
| 239 |
+
features = features.to(device)
|
| 240 |
+
exact_target = exact_target.to(device)
|
| 241 |
+
family_target = family_target.to(device)
|
| 242 |
+
if args.context_dropout:
|
| 243 |
+
drop = torch.rand(len(features), device=device) < args.context_dropout
|
| 244 |
+
features = features.clone()
|
| 245 |
+
features[drop, :, 10:15] = 0.0
|
| 246 |
+
optimizer.zero_grad(set_to_none=True)
|
| 247 |
+
with torch.no_grad():
|
| 248 |
+
online = online_adapter(features)
|
| 249 |
+
exact, family = engine.model.classify_trajectory(
|
| 250 |
+
formula_adapter(online),
|
| 251 |
+
)
|
| 252 |
+
loss = (
|
| 253 |
+
torch.nn.functional.cross_entropy(family, family_target)
|
| 254 |
+
+ args.exact_loss_weight
|
| 255 |
+
* torch.nn.functional.cross_entropy(exact, exact_target)
|
| 256 |
+
)
|
| 257 |
+
loss.backward()
|
| 258 |
+
torch.nn.utils.clip_grad_norm_(formula_adapter.parameters(), 2.0)
|
| 259 |
+
optimizer.step()
|
| 260 |
+
losses.append(float(loss.detach()))
|
| 261 |
+
scheduler.step()
|
| 262 |
+
validation_logits = _forward06(
|
| 263 |
+
engine.model, online_adapter, formula_adapter, validation_x,
|
| 264 |
+
device=device, batch_size=args.batch_size,
|
| 265 |
+
)
|
| 266 |
+
validation_metrics = _metrics06(
|
| 267 |
+
*validation_logits, validation_exact, validation_family, engine.labels,
|
| 268 |
+
)
|
| 269 |
+
row = {
|
| 270 |
+
"epoch": epoch,
|
| 271 |
+
"loss": sum(losses) / max(len(losses), 1),
|
| 272 |
+
"validation": validation_metrics,
|
| 273 |
+
}
|
| 274 |
+
history.append(row)
|
| 275 |
+
print(json.dumps(row, ensure_ascii=False), flush=True)
|
| 276 |
+
key = (
|
| 277 |
+
float(validation_metrics["family_head_top1"]),
|
| 278 |
+
float(validation_metrics["visual_family_top1"]),
|
| 279 |
+
)
|
| 280 |
+
if key > best_key:
|
| 281 |
+
best_key = key
|
| 282 |
+
best_epoch = epoch
|
| 283 |
+
best_state = deepcopy({
|
| 284 |
+
name: value.detach().cpu()
|
| 285 |
+
for name, value in formula_adapter.state_dict().items()
|
| 286 |
+
})
|
| 287 |
+
stale = 0
|
| 288 |
+
else:
|
| 289 |
+
stale += 1
|
| 290 |
+
if stale >= args.patience:
|
| 291 |
+
break
|
| 292 |
+
if best_state is None:
|
| 293 |
+
raise RuntimeError("formula adapter checkpoint가 선택되지 않았습니다.")
|
| 294 |
+
formula_adapter.load_state_dict(best_state)
|
| 295 |
+
selected_validation_logits = _forward06(
|
| 296 |
+
engine.model, online_adapter, formula_adapter, validation_x,
|
| 297 |
+
device=device, batch_size=args.batch_size,
|
| 298 |
+
)
|
| 299 |
+
selected_validation = _metrics06(
|
| 300 |
+
*selected_validation_logits,
|
| 301 |
+
validation_exact, validation_family, engine.labels,
|
| 302 |
+
)
|
| 303 |
+
if len(test_x):
|
| 304 |
+
test_logits = _forward06(
|
| 305 |
+
engine.model, online_adapter, formula_adapter, test_x,
|
| 306 |
+
device=device, batch_size=args.batch_size,
|
| 307 |
+
)
|
| 308 |
+
official_test = _metrics06(
|
| 309 |
+
*test_logits, test_exact, test_family, engine.labels,
|
| 310 |
+
)
|
| 311 |
+
else:
|
| 312 |
+
official_test = None
|
| 313 |
+
args.output.mkdir(parents=True, exist_ok=True)
|
| 314 |
+
checkpoint = args.output / "formula_adapter.pt"
|
| 315 |
+
torch.save({
|
| 316 |
+
"schema": "aiflow-math-ink-06-formula-adapter-r-v1",
|
| 317 |
+
"state_dict": best_state,
|
| 318 |
+
"hidden_size": args.hidden_size,
|
| 319 |
+
"base_checkpoint": str(base_checkpoint),
|
| 320 |
+
"online_adapter": str(args.adapter),
|
| 321 |
+
"selected_epoch": best_epoch,
|
| 322 |
+
"context_dropout": args.context_dropout,
|
| 323 |
+
"exact_loss_weight": args.exact_loss_weight,
|
| 324 |
+
"track": "R_noncommercial_only",
|
| 325 |
+
"product_validation": False,
|
| 326 |
+
"distillation_allowed": False,
|
| 327 |
+
}, checkpoint)
|
| 328 |
+
report = {
|
| 329 |
+
"experiment": "R-MATH-INK-06-FORMULA-ADAPTER-001",
|
| 330 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 331 |
+
"seed": args.seed,
|
| 332 |
+
"device": str(device),
|
| 333 |
+
"cuda_device": (
|
| 334 |
+
torch.cuda.get_device_name(device) if device.type == "cuda" else None
|
| 335 |
+
),
|
| 336 |
+
"split_contract": "CROHME trainData writer fit/validation; testDataGT one-shot",
|
| 337 |
+
"samples": {
|
| 338 |
+
"fit": len(train_x),
|
| 339 |
+
"validation": len(validation_x),
|
| 340 |
+
"official_test": len(test_x),
|
| 341 |
+
},
|
| 342 |
+
"label_support": {
|
| 343 |
+
"fit": len(Counter(train_truths)),
|
| 344 |
+
"validation": len(Counter(validation_truths)),
|
| 345 |
+
"official_test": len(Counter(test_truths)),
|
| 346 |
+
},
|
| 347 |
+
"baseline_validation": baseline_validation,
|
| 348 |
+
"selected_epoch": best_epoch,
|
| 349 |
+
"selected_validation": selected_validation,
|
| 350 |
+
"official_test": official_test,
|
| 351 |
+
"official_test_skipped": args.skip_official_test,
|
| 352 |
+
"history": history,
|
| 353 |
+
"checkpoint": checkpoint.name,
|
| 354 |
+
"checkpoint_bytes": checkpoint.stat().st_size,
|
| 355 |
+
"track": "R_noncommercial_only",
|
| 356 |
+
"product_validation": False,
|
| 357 |
+
"distillation_allowed": False,
|
| 358 |
+
}
|
| 359 |
+
(args.output / "report.json").write_text(
|
| 360 |
+
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
|
| 361 |
+
encoding="utf-8",
|
| 362 |
+
)
|
| 363 |
+
print(json.dumps({
|
| 364 |
+
"selected_epoch": best_epoch,
|
| 365 |
+
"baseline_validation": baseline_validation,
|
| 366 |
+
"selected_validation": selected_validation,
|
| 367 |
+
"official_test": official_test,
|
| 368 |
+
"product_validation": False,
|
| 369 |
+
}, ensure_ascii=False, indent=2))
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
if __name__ == "__main__":
|
| 373 |
+
main()
|