#!/usr/bin/env bash
# Runs the repo's train.py and test suite; every metric is parsed from their output.
set -o pipefail
PY=../env/bin/python

echo "KVERITAS_PHASE name=train"
$PY train.py 2>&1 | tee ../train.out
secs=$(grep -oP 'Training took \K[0-9.]+' ../train.out)
merges=$(grep -c '^merge ' ../train.out)
echo "KVERITAS_METRIC name=training_seconds value=$secs"
echo "KVERITAS_METRIC name=merges_logged value=$merges"
echo "KVERITAS_CLAIM metric=training_seconds value=$secs"

echo "KVERITAS_PHASE name=compression"
$PY - <<'PY'
from minbpe import BasicTokenizer, RegexTokenizer
text = open("tests/taylorswift.txt", encoding="utf-8").read()
n = len(text.encode("utf-8"))
for cls, name in ((BasicTokenizer, "basic"), (RegexTokenizer, "regex")):
    t = cls()
    t.load(f"models/{name}.model")
    ids = t.encode(text)
    assert t.decode(ids) == text
    print(f"KVERITAS_METRIC name=compression_{name} value={n / len(ids):.6f}")
    print(f"KVERITAS_CLAIM metric=compression_{name} value={n / len(ids):.6f}")
PY

echo "KVERITAS_PHASE name=tests"
$PY -m pytest -q tests 2>&1 | tee ../tests.out
passed=$(grep -oP '\K[0-9]+(?= passed)' ../tests.out | tail -1)
failed=$(grep -oP '\K[0-9]+(?= failed)' ../tests.out | tail -1)
echo "KVERITAS_METRIC name=tests_passed value=${passed:-0}"
echo "KVERITAS_METRIC name=tests_failed value=${failed:-0}"
echo "KVERITAS_CLAIM metric=tests_passed value=${passed:-0}"
