mirror of
https://github.com/vllm-project/vllm.git
synced 2025-10-20 14:53:52 +08:00
f3c378ffa7
Signed-off-by: Ye (Charlotte) Qi <yeq@meta.com> Signed-off-by: zhewenli <zhewenli@meta.com> Co-authored-by: Ye (Charlotte) Qi <yeq@meta.com> Co-authored-by: Ye (Charlotte) Qi <ye.charlotte.qi@gmail.com>
47 lines
1.2 KiB
Bash
Executable File
47 lines
1.2 KiB
Bash
Executable File
#!/bin/bash
|
|
# We can use this script to compute baseline accuracy on GSM for transformers.
|
|
#
|
|
# Make sure you have lm-eval-harness installed:
|
|
# pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
|
|
|
|
usage() {
|
|
echo``
|
|
echo "Runs lm eval harness on GSM8k using huggingface transformers."
|
|
echo "This pathway is intended to be used to create baselines for "
|
|
echo "our automated nm-test-accuracy workflow"
|
|
echo
|
|
echo "usage: ${0} <options>"
|
|
echo
|
|
echo " -m - huggingface stub or local directory of the model"
|
|
echo " -b - batch size to run the evaluation at"
|
|
echo " -l - limit number of samples to run"
|
|
echo " -f - number of fewshot samples to use"
|
|
echo
|
|
}
|
|
|
|
while getopts "m:b:l:f:" OPT; do
|
|
case ${OPT} in
|
|
m )
|
|
MODEL="$OPTARG"
|
|
;;
|
|
b )
|
|
BATCH_SIZE="$OPTARG"
|
|
;;
|
|
l )
|
|
LIMIT="$OPTARG"
|
|
;;
|
|
f )
|
|
FEWSHOT="$OPTARG"
|
|
;;
|
|
\? )
|
|
usage
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
lm_eval --model hf \
|
|
--model_args "pretrained=$MODEL,parallelize=True" \
|
|
--tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \
|
|
--batch_size "$BATCH_SIZE"
|