veil-pii-ko-small-3 / eval /speed_same_cpu.py
kakao1513's picture
Create speed_same_cpu.py
0c10452 verified
Raw History Blame Contribute Delete
3.08 kB
"""Repeatable, same-pipeline CPU comparison of the published student and teacher."""
import hashlib, json, os, platform, statistics, sys, time
from pathlib import Path
import numpy as np
import torch, transformers, numba
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent
sys.path[:0] = [str(ROOT / 'upstream'), str(ROOT / 'student8n256')]
import eval.span_f1 as upstream
from fast_decode import decode_fast
from train.labels import load_labels
torch.set_num_threads(8)
torch.set_num_interop_threads(1)
upstream.decode = lambda lp, T, s, e: decode_fast(np.ascontiguousarray(lp), s, e).tolist()
teacher = upstream.Predictor(str(ROOT / 'model'), device='cpu', max_len=128, stride=16)
student = upstream.Predictor(str(HERE / 'hf_release'), device='cpu', max_len=128, stride=16)
_, tags, _, _ = load_labels()
assert teacher.tags[:13] == tags[:13] and student.model.config.num_labels == 13
student.tags = student.tags[:13]
student.T = student.T[:13, :13]
student.s0 = np.ascontiguousarray(student.s0[:13])
student.e0 = np.ascontiguousarray(student.e0[:13])
sample = '์•ˆ๋…•ํ•˜์„ธ์š”. ๊ณ ๊ฐ ๊น€์ฒ ์ˆ˜์˜ ์—ฐ๋ฝ์ฒ˜๋Š” 010-1234-5678์ด๊ณ  ์ด๋ฉ”์ผ์€ minsu.kim@example.com์ž…๋‹ˆ๋‹ค. ์„œ์šธ์‹œ ๊ฐ•๋‚จ๊ตฌ ํ…Œํ—ค๋ž€๋กœ 123์œผ๋กœ ์„œ๋ฅ˜๋ฅผ ๋ณด๋‚ด ์ฃผ์„ธ์š”. ์ƒ๋‹ด ๋‚ด์šฉ์„ ํ™•์ธํ•œ ๋’ค ๋‹ด๋‹น์ž๊ฐ€ ๋‹ค์‹œ ์—ฐ๋ฝ๋“œ๋ฆฌ๊ฒ ์Šต๋‹ˆ๋‹ค.\n'
def document(n): return (sample * (n // len(sample) + 1))[:n]
def run(p, text):
spans = p.predict(text, bs=16)
spans = [s for s in spans if s['label'] in ('PERSON', 'ADDRESS', 'ORGANIZATION')]
return json.dumps({'spans': spans}, ensure_ascii=False).encode('utf8')
for p in (teacher, student):
for _ in range(3): run(p, document(1000))
out = {'method': 'same upstream Predictor, PyTorch FP32, filtering to 3 labels after decoding; includes UTF-8 JSON serialization; model load excluded',
'platform': platform.platform(), 'cpu': platform.processor(), 'affinity': sorted(os.sched_getaffinity(0)),
'torch': torch.__version__, 'transformers': transformers.__version__, 'numpy': np.__version__, 'numba': numba.__version__,
'threads': 8, 'interop_threads': 1, 'window': 128, 'overlap': 16, 'batch': 16, 'skip_ids': False,
'warmup_per_model': 3, 'repetitions': 3, 'rows': []}
target = HERE / 'hf_release' / 'eval' / 'speed_same_cpu.json'
for n in (100, 1000, 5000, 10000, 50000, 100000, 200000):
text = document(n)
row = {'chars': n, 'sha256_utf8': hashlib.sha256(text.encode()).hexdigest(), 'teacher': [], 'student': []}
for i in range(3):
for name, model in ((('teacher', teacher), ('student', student)) if i % 2 == 0 else (('student', student), ('teacher', teacher))):
start = time.perf_counter(); payload = run(model, text); elapsed = time.perf_counter() - start
row[name].append(elapsed)
print(n, i + 1, name, round(elapsed, 6), flush=True)
for name in ('teacher', 'student'): row[name + '_median'] = statistics.median(row[name])
out['rows'].append(row)
target.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding='utf8')