#!/usr/bin/env bash
# transformers/run_whisper.sh as committed: openai/whisper-large-v3-turbo, batch 64, the seven English test sets of
# hf-audio/open-asr-leaderboard, then normalizer/eval_utils.score_results. Data and weights cached beforehand, run offline.
ROW=/var/tmp/kv-repro/043-huggingface-open_asr_leaderboard
export HOME=$ROW/data/home HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1
export PYTHONPATH="$PYTHONPATH" LD_LIBRARY_PATH=$(python -c "import site; print(site.getsitepackages()[0])")/nvidia/npp/lib
cd transformers
echo "KVERITAS_PHASE name=evaluate"
bash run_whisper.sh 2>&1 | grep -v "it/s\]\|s/it\]\|^Map\|^Filter" | tee $ROW/data/eval.log
python - $ROW/data/eval.log <<'PY'
import re, sys
text = open(sys.argv[1]).read()
blocks = re.split(r"\*{100}\nEvaluating ", text)[1:]
wers, n = [], 0
for b in blocks:
    m = re.match(r"\S+ on \S+ / (\S+) / (\S+)", b)
    r = re.search(r"WER: ([0-9.]+) % RTFx: ([0-9.]+)", b)
    if not (m and r):
        continue
    name = re.sub(r"\W+", "_", f"{m.group(1)}_{m.group(2)}").strip("_")
    print(f"KVERITAS_METRIC name={name}_wer value={r.group(1)}")
    print(f"KVERITAS_METRIC name={name}_rtfx value={r.group(2)}")
    print(f"KVERITAS_CLAIM metric={name}_wer value={r.group(1)}")
    wers.append(float(r.group(1)))
if len(wers) != 7:
    sys.exit(f"expected 7 datasets, parsed {len(wers)}")
print(f"KVERITAS_CLAIM metric=mean_wer_7_sets value={sum(wers) / len(wers):.2f}")
PY
