#!/usr/bin/env bash
# benchmark/: wer_benchmark.py (large-v3, LibriSpeech clean validation, all 2703 utterances, streamed),
# speed_benchmark.py (benchmark.m4a, 3 repeats of 10) and memory_benchmark.py --gpu_memory, as written.
ROW=/var/tmp/kv-repro/042-SYSTRAN-faster-whisper
export HOME=$ROW/data/home
SP=$(python -c "import site; print(site.getsitepackages()[0])")
export LD_LIBRARY_PATH=$SP/nvidia/cublas/lib:$SP/nvidia/cudnn/lib
cd benchmark
echo "KVERITAS_PHASE name=wer"
python wer_benchmark.py 2>&1 | grep -v "Evaluating\.\.\.:" | grep -v "^Extension modules\|^  File\|^Thread\|<no Python frame>" | tee $ROW/data/wer.log
wer=$(grep -oP "^WER: \K[0-9.]+" $ROW/data/wer.log) || exit 1
echo "KVERITAS_METRIC name=librispeech_clean_val_wer value=$wer"
echo "KVERITAS_CLAIM metric=librispeech_clean_val_wer value=$wer"
echo "KVERITAS_PHASE name=speed"
python speed_benchmark.py 2>&1 | tail -3 | tee $ROW/data/speed.log
t=$(grep -oP "Min execution time: \K[0-9.]+" $ROW/data/speed.log) || exit 1
echo "KVERITAS_CLAIM metric=benchmark_m4a_min_transcribe_s value=$t"
echo "KVERITAS_PHASE name=memory"
python memory_benchmark.py --gpu_memory 2>&1 | grep -v "^\[" | tail -6 | tee $ROW/data/mem.log
m=$(grep -oP "Maximum GPU memory usage: \K[0-9]+" $ROW/data/mem.log) || exit 1
echo "KVERITAS_CLAIM metric=benchmark_m4a_max_gpu_mem_mib value=$m"
