#!/usr/bin/env bash
# demo_watermark.py with facebook/opt-1.3b, default settings (gamma 0.25, delta 2.0, simple_1 seeding, z threshold 4.0),
# once per generation seed. Each run generates 200 tokens with and without the watermark and detects both.
set -o pipefail
ROW=/var/tmp/kv-repro/039-jwkirchenbauer-lm-watermarking
export HOME=$ROW/data/home
for seed in 123 1 2 3 4; do
  echo "KVERITAS_PHASE name=seed_$seed"
  echo "KVERITAS_INPUT src=seed:$seed"
  python demo_watermark.py --model_name_or_path facebook/opt-1.3b --run_gradio False --generation_seed $seed > $ROW/data/seed_$seed.log 2>&1 || { tail -20 $ROW/data/seed_$seed.log; exit 1; }
  grep -v "^ Namespace\|^Namespace" $ROW/data/seed_$seed.log
  python - $ROW/data/seed_$seed.log $seed <<'PY'
import re, sys
text, seed = open(sys.argv[1]).read(), sys.argv[2]
for arm in ("without", "with"):
    block = text.split(f"Output {arm} watermark:")[1].split("Namespace(")[0]
    z = re.search(r"\['z-score', '([^']+)'\]", block).group(1)
    g = re.search(r"\['Fraction of T in Greenlist', '([^']+)%'\]", block).group(1)
    name = "plain" if arm == "without" else "watermarked"
    print(f"KVERITAS_METRIC name={name}_z_score value={z} step={seed}")
    print(f"KVERITAS_METRIC name={name}_green_fraction_pct value={g} step={seed}")
    print(f"KVERITAS_CLAIM metric={name}_z_score_seed_{seed} value={z}")
PY
done
