#!/usr/bin/env bash
# TabM on California housing: re-run the paper's 15 evaluation seeds with the shipped configs
# (paper/exp/tabm/california/0-evaluation/<seed>.toml) and compare with the reports stored in the repo.
set -o pipefail
cd paper || exit 1
export CUDA_VISIBLE_DEVICES=0 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1
SRC=exp/tabm/california/0-evaluation
OUT=exp/kv-reproduce
rm -rf "$OUT" && mkdir -p "$OUT"
for s in $(seq 0 14); do
  echo "KVERITAS_PHASE name=seed_$s"
  cp "$SRC/$s.toml" "$OUT/$s.toml"
  python bin/model.py "$OUT/$s.toml" --force 2>&1 | tail -3 || exit 1
done
python - "$SRC" "$OUT" <<'PY'
import json, statistics, sys
src, out = sys.argv[1], sys.argv[2]
ours, paper = [], []
for s in range(15):
    o = json.load(open(f"{out}/{s}/report.json"))["metrics"]["test"]["rmse"]
    p = json.load(open(f"{src}/{s}/report.json"))["metrics"]["test"]["rmse"]
    ours.append(o); paper.append(p)
    print(f"KVERITAS_METRIC name=test_rmse value={o:.6g} step={s}")
    print(f"KVERITAS_METRIC name=paper_test_rmse value={p:.6g} step={s}")
print(f"KVERITAS_METRIC name=test_rmse_std value={statistics.stdev(ours):.6g}")
print(f"KVERITAS_CLAIM metric=test_rmse_mean value={statistics.mean(ours):.6g}")
print(f"KVERITAS_CLAIM metric=paper_test_rmse_mean value={statistics.mean(paper):.6g}")
PY
