#!/usr/bin/env bash
# CLIP_benchmark zero-shot classification: OpenCLIP ViT-B-32 laion2b_s34b_b79k on CIFAR-10, CIFAR-100, STL-10.
set -o pipefail
cd "$(dirname "$0")"
RES=../data/res
rm -rf "$RES"; mkdir -p "$RES"
export TQDM_DISABLE=1
for DS in cifar10 cifar100 stl10; do
  echo "KVERITAS_PHASE name=zeroshot_$DS"
  ../env/bin/clip_benchmark eval --model ViT-B-32 --pretrained laion2b_s34b_b79k --dataset "$DS" \
    --dataset_root "../data/{dataset}" --task zeroshot_classification \
    --output "$RES/{dataset}_{pretrained}_{model}_{language}_{task}.json" 2>&1 || exit 1
  ../env/bin/python - "$RES/${DS}_laion2b_s34b_b79k_ViT-B-32_en_zeroshot_classification.json" <<'PY'
import json, sys
r = json.load(open(sys.argv[1]))
ds, m = r["dataset"], r["metrics"]
for k in ("acc1", "acc5", "mean_per_class_recall"):
    print(f"KVERITAS_METRIC name={ds}_{k} value={m[k]}")
for k in ("acc1", "acc5"):
    print(f"KVERITAS_CLAIM metric={ds}_{k} value={m[k]}")
PY
done
