[benchmark] fix benchmark script (#8648)

* add llama-7b_auto_dp2mp2pp2 benchmark sh

* add llama-7b_auto_dp2mp2pp2 benchmark script for cinn

* update llama-7b_auto_dp2mp2pp2 benchmark script

* Update run_benchmark.sh

* Update run_benchmark.sh

* fix llama-7b_auto_dp2mp2pp2/benchmark_common

* Update run_benchmark.sh

* Update prepare.sh

* Update prepare.sh

* Update prepare.sh

* Update prepare.sh

* Update prepare.sh

* fix benchmark script

* fix script path for transformer benchmark

Author: mmglove

tests/test_tipc/benchmark/modules/ernie_tiny.py

@@ -30,7 +30,9 @@
 sys.path.insert(
     0,
     os.path.abspath(
-        os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, "model_zoo", "ernie-3.0")
+        os.path.join(
+            os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, "legacy", "model_zoo", "ernie-3.0"
+        )
     ),
 )
 

tests/test_tipc/benchmark/options.py

@@ -25,15 +25,12 @@
 )
 from .modules.lr_scheduler import *  # noqa: F403
 from .modules.optimizer import *  # noqa: F403
-from .modules.rnnlm import RNNLMBenchmark
-from .modules.seq2seq import Seq2SeqBenchmark
 
 try:
     from .modules.stablediffusion import StableDiffusionBenchmark
 except Exception:
     StableDiffusionBenchmark = None
 from .modules.t5_for_conditional_generation import T5ForConditionalGenerationBenchmark
-from .modules.xlnet import XLNetBenchmark
 
 __all__ = [
     "MODEL_REGISTRY",
@@ -44,10 +41,7 @@
 ]
 
 MODEL_REGISTRY = {
-    "seq2seq": Seq2SeqBenchmark,
-    "xlnet": XLNetBenchmark,
     "lac": BiGruCrfBenchmark,
-    "ptb": RNNLMBenchmark,
     "ernie_tiny": ErnieTinyBenchmark,
     "ernie3_for_sequence_classification": Ernie3ForSequenceClassificationBenchmark,
     "bert_for_question_answering": BertForQuestionAnsweringBenchmark,

tests/test_tipc/configs/bert/base/bert_base_seqlen128_cinn_train_infer_python.txt

@@ -13,7 +13,7 @@ null:null
 null:null
 ##
 trainer:norm_train
-norm_train:../examples/language_model/bert/run_pretrain.py --preprocessing_num_workers 8 --max_predictions_per_seq 20 --learning_rate 1e-4 --weight_decay 1e-2 --adam_epsilon 1e-6 --warmup_steps 10000 --output_dir ./tmp2/ --logging_steps 10 --save_steps 20000 --model_type bert --model_name_or_path bert-base-uncased --input_dir ./data/wikicorpus_en_seqlen128 --fuse_transformer false --amp_level O2 --cinn True
+norm_train:../legacy/model_zoo/bert/run_pretrain.py --preprocessing_num_workers 8 --max_predictions_per_seq 20 --learning_rate 1e-4 --weight_decay 1e-2 --adam_epsilon 1e-6 --warmup_steps 10000 --output_dir ./tmp2/ --logging_steps 10 --save_steps 20000 --model_type bert --model_name_or_path bert-base-uncased --input_dir ./data/wikicorpus_en_seqlen128 --fuse_transformer false --amp_level O2 --cinn True
 pact_train:null
 fpgm_train:null
 distill_train:null

tests/test_tipc/configs/bert/base/train_infer_python.txt

@@ -13,7 +13,7 @@ null:null
 null:null
 ##
 trainer:norm_train
-norm_train:../examples/language_model/bert/run_pretrain.py --max_predictions_per_seq 20 --learning_rate 1e-4 --weight_decay 1e-2 --adam_epsilon 1e-6 --warmup_steps 10000 --output_dir ./tmp2/ --logging_steps 10 --save_steps 20000 --model_type bert --model_name_or_path bert-base-uncased --input_dir ./data/wikicorpus_en_seqlen128 --fuse_transformer true --amp_level O2
+norm_train:../legacy/model_zoo/bert/run_pretrain.py --max_predictions_per_seq 20 --learning_rate 1e-4 --weight_decay 1e-2 --adam_epsilon 1e-6 --warmup_steps 10000 --output_dir ./tmp2/ --logging_steps 10 --save_steps 20000 --model_type bert --model_name_or_path bert-base-uncased --input_dir ./data/wikicorpus_en_seqlen128 --fuse_transformer true --amp_level O2
 pact_train:null
 fpgm_train:null
 distill_train:null

tests/test_tipc/configs/bert/large/train_infer_python.txt

@@ -13,7 +13,7 @@ null:null
 null:null
 ##
 trainer:norm_train
-norm_train:../examples/language_model/bert/run_pretrain.py --max_predictions_per_seq 20 --learning_rate 1e-4 --weight_decay 1e-2 --adam_epsilon 1e-6 --warmup_steps 10000 --output_dir ./tmp2/ --logging_steps 10 --save_steps 20000 --model_type bert --model_name_or_path bert-large-uncased --input_dir ./data/wikicorpus_en_seqlen512
+norm_train:../legacy/model_zoo/bert/run_pretrain.py --max_predictions_per_seq 20 --learning_rate 1e-4 --weight_decay 1e-2 --adam_epsilon 1e-6 --warmup_steps 10000 --output_dir ./tmp2/ --logging_steps 10 --save_steps 20000 --model_type bert --model_name_or_path bert-large-uncased --input_dir ./data/wikicorpus_en_seqlen512
 pact_train:null
 fpgm_train:null
 distill_train:null

tests/test_tipc/configs/gpt2/train_infer_python.txt

@@ -13,7 +13,7 @@ null:null
 null:null
 ##
 trainer:norm_train
-norm_train:../examples/language_model/gpt/run_pretrain.py --model_type="gpt" --model_name_or_path="gpt2-en" --save_steps 100000 --decay_steps 320000 --weight_decay 0.01 --warmup_rate 0.01 --grad_clip 1.0 --logging_freq 1 --eval_freq 1000 --device "gpu" --min_lr 0.00001 --max_lr 0.00015 --max_seq_len 1024 --output_dir=./output/ --input_dir=./data/
+norm_train:../legacy/model_zoo/gpt/run_pretrain.py --model_type="gpt" --model_name_or_path="gpt2-en" --save_steps 100000 --decay_steps 320000 --weight_decay 0.01 --warmup_rate 0.01 --grad_clip 1.0 --logging_freq 1 --eval_freq 1000 --device "gpu" --min_lr 0.00001 --max_lr 0.00015 --max_seq_len 1024 --output_dir=./output/ --input_dir=./data/
 pact_train:null
 fpgm_train:null
 distill_train:null

tests/test_tipc/configs/gpt3/train_infer_python.txt

@@ -13,7 +13,7 @@ null:null
 null:null
 ##
 trainer:norm_train
-norm_train:../examples/language_model/gpt-3/dygraph/run_pretrain.py --model_type="gpt" --model_name_or_path="gpt2-en" --save_steps 100000 --decay_steps 320000 --weight_decay 0.01 --warmup_rate 0.01 --grad_clip 1.0 --logging_freq 1 --eval_freq 1000 --device "gpu" --min_lr 0.00001 --max_lr 0.00015 --max_seq_len 1024 --output_dir=./output/ --input_dir=./data/
+norm_train:../legacy/model_zoo/gpt-3/dygraph/run_pretrain.py --model_type="gpt" --model_name_or_path="gpt2-en" --save_steps 100000 --decay_steps 320000 --weight_decay 0.01 --warmup_rate 0.01 --grad_clip 1.0 --logging_freq 1 --eval_freq 1000 --device "gpu" --min_lr 0.00001 --max_lr 0.00015 --max_seq_len 1024 --output_dir=./output/ --input_dir=./data/
 pact_train:null
 fpgm_train:null
 distill_train:null

tests/test_tipc/configs/rnnlm/train_infer_python.txt

@@ -13,7 +13,7 @@ null:null
 null:null
 ##
 trainer:norm_train
-norm_train:../examples/machine_translation/transformer/train.py --config ../examples/machine_translation/transformer/configs/transformer.base.yaml --train_file ../examples/machine_translation/transformer/train.en ../examples/machine_translation/transformer/train.de --dev_file ../examples/machine_translation/transformer/dev.en ../examples/machine_translation/transformer/dev.de --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
+norm_train:../legacy/examples/machine_translation/transformer/train.py --config ../examples/machine_translation/transformer/configs/transformer.base.yaml --train_file ../examples/machine_translation/transformer/train.en ../examples/machine_translation/transformer/train.de --dev_file ../examples/machine_translation/transformer/dev.en ../examples/machine_translation/transformer/dev.de --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
 pact_train:null
 fpgm_train:null
 distill_train:null

tests/test_tipc/configs/seq2seq/train_infer_python.txt

@@ -13,21 +13,21 @@ null:null
 null:null
 ##
 trainer:norm_train
-norm_train:../examples/machine_translation/transformer/train.py --config ../examples/machine_translation/transformer/configs/transformer.base.yaml --train_file ../examples/machine_translation/transformer/train.en ../examples/machine_translation/transformer/train.de --dev_file ../examples/machine_translation/transformer/dev.en ../examples/machine_translation/transformer/dev.de --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
+norm_train:../legacy/examples/machine_translation/transformer/train.py --config ../legacy/examples/machine_translation/transformer/configs/transformer.base.yaml --train_file ../legacy/examples/machine_translation/transformer/train.en ../legacy/examples/machine_translation/transformer/train.de --dev_file ../legacy/examples/machine_translation/transformer/dev.en ../legacy/examples/machine_translation/transformer/dev.de --vocab_file ../legacy/examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
 pact_train:null
 fpgm_train:null
 distill_train:null
 null:null
 null:null
 ##
 ===========================eval_params=========================== 
-eval:../examples/machine_translation/transformer/predict.py --config ../examples/machine_translation/transformer/configs/transformer.base.yaml --test_file ../examples/machine_translation/transformer/test.en ../examples/machine_translation/transformer/test.de --without_ft --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
+eval:./legacy/examples/machine_translation/transformer/predict.py --config ./legacy/examples/machine_translation/transformer/configs/transformer.base.yaml --test_file ./legacy/examples/machine_translation/transformer/test.en ./legacy/examples/machine_translation/transformer/test.de --without_ft --vocab_file ./legacy/examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
 null:null
 ##
 ===========================infer_params===========================
 null:null
 null:null
-norm_export:../examples/machine_translation/transformer/export_model.py --config ../examples/machine_translation/transformer/configs/transformer.base.yaml --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --bos_token "<s>" --eos_token "<e>" --benchmark
+norm_export:./legacy/examples/machine_translation/transformer/export_model.py --config ./legacy/examples/machine_translation/transformer/configs/transformer.base.yaml --vocab_file ./legacy/examples/machine_translation/transformer/vocab_all.bpe.33712 --bos_token "<s>" --eos_token "<e>" --benchmark
 quant_export:null
 fpgm_export:null
 distill_export:null
@@ -37,7 +37,7 @@ export2:null
 infer_model:null
 infer_export:null
 infer_quant:null
-inference:../examples/machine_translation/transformer/deploy/python/inference.py --config ../examples/machine_translation/transformer/configs/transformer.base.yaml --profile --test_file ../examples/machine_translation/transformer/test.en ../examples/machine_translation/transformer/test.de  --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
+inference:../legacy/examples/machine_translation/transformer/deploy/python/inference.py --config ../legacy/examples/machine_translation/transformer/configs/transformer.base.yaml --profile --test_file ../legacy/examples/machine_translation/transformer/test.en ../legacy/examples/machine_translation/transformer/test.de  --vocab_file ../legacy/examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
 --device:gpu|cpu
 --use_mkl:True
 --threads:1|6

tests/test_tipc/configs/transformer/base/train_dy2static_python.txt

@@ -13,7 +13,7 @@ null:null
 null:null
 ##
 trainer:norm_train
-norm_train:../examples/machine_translation/transformer/static/train.py --config ../examples/machine_translation/transformer/configs/transformer.base.yaml --distributed --train_file ../examples/machine_translation/transformer/train.en ../examples/machine_translation/transformer/train.de --dev_file ../examples/machine_translation/transformer/dev.en ../examples/machine_translation/transformer/dev.de  --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33708 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>"
+norm_train:../legacy/examples/machine_translation/transformer/static/train.py --config ../examples/machine_translation/transformer/configs/transformer.base.yaml --distributed --train_file ../examples/machine_translation/transformer/train.en ../examples/machine_translation/transformer/train.de --dev_file ../examples/machine_translation/transformer/dev.en ../examples/machine_translation/transformer/dev.de  --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33708 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>"
 pact_train:null
 fpgm_train:null
 distill_train:null

tests/test_tipc/configs/transformer/base/train_infer_python.txt

@@ -13,7 +13,7 @@ null:null
 null:null
 ##
 trainer:norm_train
-norm_train:../examples/machine_translation/transformer/train.py --config ../examples/machine_translation/transformer/configs/transformer.big.yaml --train_file ../examples/machine_translation/transformer/train.en ../examples/machine_translation/transformer/train.de --dev_file ../examples/machine_translation/transformer/dev.en ../examples/machine_translation/transformer/dev.de --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
+norm_train:../legacy/examples/machine_translation/transformer/train.py --config ../examples/machine_translation/transformer/configs/transformer.big.yaml --train_file ../examples/machine_translation/transformer/train.en ../examples/machine_translation/transformer/train.de --dev_file ../examples/machine_translation/transformer/dev.en ../examples/machine_translation/transformer/dev.de --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
 pact_train:null
 fpgm_train:null
 distill_train:null

tests/test_tipc/configs/transformer/base/transformer_base_static_params.txt

@@ -13,21 +13,21 @@ null:null
 null:null
 ##
 trainer:norm_train
-norm_train:../examples/machine_translation/transformer/train.py --config ../examples/machine_translation/transformer/configs/transformer.big.yaml --train_file ../examples/machine_translation/transformer/train.en ../examples/machine_translation/transformer/train.de --dev_file ../examples/machine_translation/transformer/dev.en ../examples/machine_translation/transformer/dev.de --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
+norm_train:../legacy/examples/machine_translation/transformer/train.py --config ../legacy/examples/machine_translation/transformer/configs/transformer.big.yaml --train_file ../legacy/examples/machine_translation/transformer/train.en ../legacy/examples/machine_translation/transformer/train.de --dev_file ../legacy/examples/machine_translation/transformer/dev.en ../legacy/examples/machine_translation/transformer/dev.de --vocab_file ../legacy/examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
 pact_train:null
 fpgm_train:null
 distill_train:null
 null:null
 null:null
 ##
 ===========================eval_params=========================== 
-eval:../examples/machine_translation/transformer/predict.py --config ../examples/machine_translation/transformer/configs/transformer.big.yaml --test_file ../examples/machine_translation/transformer/test.en ../examples/machine_translation/transformer/test.de --without_ft --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
+eval:../legacy/examples/machine_translation/transformer/predict.py --config ../legacy/examples/machine_translation/transformer/configs/transformer.big.yaml --test_file ../legacy/examples/machine_translation/transformer/test.en ../legacy/examples/machine_translation/transformer/test.de --without_ft --vocab_file ../legacy/examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
 null:null
 ##
 ===========================infer_params===========================
 null:null
 null:null
-norm_export:../examples/machine_translation/transformer/export_model.py --config ../examples/machine_translation/transformer/configs/transformer.big.yaml --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --bos_token "<s>" --eos_token "<e>" --benchmark
+norm_export:../legacy/examples/machine_translation/transformer/export_model.py --config ../legacy/examples/machine_translation/transformer/configs/transformer.big.yaml --vocab_file ../legacy/examples/machine_translation/transformer/vocab_all.bpe.33712 --bos_token "<s>" --eos_token "<e>" --benchmark
 quant_export:null
 fpgm_export:null
 distill_export:null
@@ -37,7 +37,7 @@ export2:null
 infer_model:null
 infer_export:null
 infer_quant:null
-inference:../examples/machine_translation/transformer/deploy/python/inference.py --config ../examples/machine_translation/transformer/configs/transformer.big.yaml --profile --test_file ../examples/machine_translation/transformer/test.en ../examples/machine_translation/transformer/test.de  --vocab_file ../examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
+inference:../legacy/examples/machine_translation/transformer/deploy/python/inference.py --config ../legacy/examples/machine_translation/transformer/configs/transformer.big.yaml --profile --test_file ../legacy/examples/machine_translation/transformer/test.en ../legacy/examples/machine_translation/transformer/test.de  --vocab_file ../legacy/examples/machine_translation/transformer/vocab_all.bpe.33712 --unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" --benchmark
 --device:gpu|cpu
 --use_mkl:True
 --threads:1|6