#!/usr/bin/env bash
# README example: lighteval accelerate on SmolLM2-1.7B-Instruct, ARC-Challenge (25-shot) and HellaSwag
# (10-shot), full test sets, batch size 8. A fresh cache directory makes every sample run in this session.
# Metrics are read from the results JSON lighteval writes.
set -o pipefail
ROW=/var/tmp/kv-repro/029-huggingface-lighteval
export HOME=$ROW/data/home
CACHE=$ROW/data/cache-$(date +%s)
OUT=$ROW/data/results-$(date +%s)
cd "$ROW/data" || exit 1
echo "KVERITAS_PHASE name=evaluate"
lighteval accelerate "model_name=SmolLM2-1.7B-Instruct,batch_size=8,cache_dir=$CACHE" \
  "arc:challenge|25,hellaswag|10" --output-dir "$OUT" 2>&1 | grep -E "^\||WARNING.*max" || exit 1
python - "$OUT" <<'PY'
import glob, json, sys
path = sorted(glob.glob(f"{sys.argv[1]}/results/**/results_*.json", recursive=True))[-1]
r = json.load(open(path))["results"]
for task, metrics in r.items():
    if task == "all":
        continue
    name = task.split("|")[0].replace(":", "_")
    for k, v in metrics.items():
        if isinstance(v, (int, float)) and not k.endswith("_stderr"):
            print(f"KVERITAS_METRIC name={name}_{k} value={v:.6g}")
            print(f"KVERITAS_CLAIM metric={name}_{k} value={v:.6g}")
PY
