#!/usr/bin/env bash
# README benchmark: llama-bench (pp512, tg128) and WikiText-2 perplexity, Qwen2.5-7B-Instruct Q4_K_M, all layers on the GPU.
# Built before the session with -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=120.
set -o pipefail
ROW=/var/tmp/kv-repro/016-ggml-org-llama.cpp
M=$ROW/data/qwen2.5-7b-instruct-q4_k_m-00001-of-00002.gguf
echo "KVERITAS_PHASE name=bench"
build/bin/llama-bench -m "$M" -o json > "$ROW/bench.json" || exit 1
cat "$ROW/bench.json"
python - "$ROW/bench.json" <<'PY'
import json, sys
for t in json.load(open(sys.argv[1])):
    name = "pp512" if t["n_prompt"] and not t["n_gen"] else "tg128"
    print(f"KVERITAS_METRIC name={name}_tokens_per_second value={t['avg_ts']:.6g}")
    print(f"KVERITAS_METRIC name={name}_stddev value={t['stddev_ts']:.6g}")
    print(f"KVERITAS_CLAIM metric={name}_tokens_per_second value={t['avg_ts']:.6g}")
PY
echo "KVERITAS_PHASE name=perplexity"
build/bin/llama-perplexity -m "$M" -f "$ROW/data/wikitext-2-raw/wiki.test.raw" -ngl 99 2>&1 | tee "$ROW/ppl.out" || exit 1
python - "$ROW/ppl.out" <<'PY'
import re, sys
text = open(sys.argv[1]).read()
for i, v in re.findall(r"\[(\d+)\]([0-9.]+)", text):
    if int(i) % 20 == 0:
        print(f"KVERITAS_METRIC name=running_ppl value={v} step={i}")
m = re.search(r"Final estimate: PPL = ([0-9.]+) \+/- ([0-9.]+)", text)
print(f"KVERITAS_METRIC name=ppl_stderr value={m.group(2)}")
print(f"KVERITAS_CLAIM metric=wikitext2_ppl value={m.group(1)}")
PY
