diff --git a/examples/machine_translation/transformer/deploy/python/README.md b/legacy/examples/machine_translation/transformer/deploy/python/README.md
similarity index 100%
rename from examples/machine_translation/transformer/deploy/python/README.md
rename to legacy/examples/machine_translation/transformer/deploy/python/README.md
diff --git a/examples/machine_translation/transformer/deploy/python/benchmark.sh b/legacy/examples/machine_translation/transformer/deploy/python/benchmark.sh
similarity index 64%
rename from examples/machine_translation/transformer/deploy/python/benchmark.sh
rename to legacy/examples/machine_translation/transformer/deploy/python/benchmark.sh
index 0b9b8c482995..4d953a467f51 100644
--- a/examples/machine_translation/transformer/deploy/python/benchmark.sh
+++ b/legacy/examples/machine_translation/transformer/deploy/python/benchmark.sh
@@ -1,4 +1,19 @@
#!/bin/bash
+
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
model_dir=${1}
model=${2}
mkdir -p output_pipeline
diff --git a/examples/machine_translation/transformer/deploy/python/inference.py b/legacy/examples/machine_translation/transformer/deploy/python/inference.py
similarity index 100%
rename from examples/machine_translation/transformer/deploy/python/inference.py
rename to legacy/examples/machine_translation/transformer/deploy/python/inference.py
diff --git a/examples/machine_translation/transformer/deploy/python/tls/benchmark_utils.py b/legacy/examples/machine_translation/transformer/deploy/python/tls/benchmark_utils.py
similarity index 100%
rename from examples/machine_translation/transformer/deploy/python/tls/benchmark_utils.py
rename to legacy/examples/machine_translation/transformer/deploy/python/tls/benchmark_utils.py
diff --git a/examples/machine_translation/transformer/deploy/python/tls/recorder.py b/legacy/examples/machine_translation/transformer/deploy/python/tls/recorder.py
similarity index 100%
rename from examples/machine_translation/transformer/deploy/python/tls/recorder.py
rename to legacy/examples/machine_translation/transformer/deploy/python/tls/recorder.py
diff --git a/examples/machine_translation/transformer/deploy/serving/README.md b/legacy/examples/machine_translation/transformer/deploy/serving/README.md
similarity index 100%
rename from examples/machine_translation/transformer/deploy/serving/README.md
rename to legacy/examples/machine_translation/transformer/deploy/serving/README.md
diff --git a/examples/machine_translation/transformer/deploy/serving/benchmark.py b/legacy/examples/machine_translation/transformer/deploy/serving/benchmark.py
similarity index 100%
rename from examples/machine_translation/transformer/deploy/serving/benchmark.py
rename to legacy/examples/machine_translation/transformer/deploy/serving/benchmark.py
diff --git a/examples/machine_translation/transformer/deploy/serving/benchmark_serving.sh b/legacy/examples/machine_translation/transformer/deploy/serving/benchmark_serving.sh
similarity index 71%
rename from examples/machine_translation/transformer/deploy/serving/benchmark_serving.sh
rename to legacy/examples/machine_translation/transformer/deploy/serving/benchmark_serving.sh
index 7f57ef5a0873..4b5f686b2f47 100644
--- a/examples/machine_translation/transformer/deploy/serving/benchmark_serving.sh
+++ b/legacy/examples/machine_translation/transformer/deploy/serving/benchmark_serving.sh
@@ -1,3 +1,17 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
modelname="transformer"
export FLAGS_profile_pipeline=1
# HTTP
diff --git a/examples/machine_translation/transformer/deploy/serving/export_serving_model.py b/legacy/examples/machine_translation/transformer/deploy/serving/export_serving_model.py
similarity index 56%
rename from examples/machine_translation/transformer/deploy/serving/export_serving_model.py
rename to legacy/examples/machine_translation/transformer/deploy/serving/export_serving_model.py
index 97e0a526dc80..feb38dda199e 100644
--- a/examples/machine_translation/transformer/deploy/serving/export_serving_model.py
+++ b/legacy/examples/machine_translation/transformer/deploy/serving/export_serving_model.py
@@ -1,4 +1,19 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
import argparse
+
import paddle
import paddle_serving_client.io as serving_io
diff --git a/examples/machine_translation/transformer/deploy/serving/transformer_reader.py b/legacy/examples/machine_translation/transformer/deploy/serving/transformer_reader.py
similarity index 73%
rename from examples/machine_translation/transformer/deploy/serving/transformer_reader.py
rename to legacy/examples/machine_translation/transformer/deploy/serving/transformer_reader.py
index 2b295e7d6c37..b613a5906138 100644
--- a/examples/machine_translation/transformer/deploy/serving/transformer_reader.py
+++ b/legacy/examples/machine_translation/transformer/deploy/serving/transformer_reader.py
@@ -1,7 +1,21 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
import numpy as np
-from paddlenlp.datasets import load_dataset
from paddlenlp.data import Pad, Vocab
+from paddlenlp.datasets import load_dataset
class TransformerReader(object):
diff --git a/examples/machine_translation/transformer/deploy/serving/transformer_web_client.py b/legacy/examples/machine_translation/transformer/deploy/serving/transformer_web_client.py
similarity index 100%
rename from examples/machine_translation/transformer/deploy/serving/transformer_web_client.py
rename to legacy/examples/machine_translation/transformer/deploy/serving/transformer_web_client.py
diff --git a/examples/machine_translation/transformer/deploy/serving/transformer_web_server.py b/legacy/examples/machine_translation/transformer/deploy/serving/transformer_web_server.py
similarity index 100%
rename from examples/machine_translation/transformer/deploy/serving/transformer_web_server.py
rename to legacy/examples/machine_translation/transformer/deploy/serving/transformer_web_server.py
diff --git a/examples/machine_translation/transformer/deploy/serving/utils/recorder.py b/legacy/examples/machine_translation/transformer/deploy/serving/utils/recorder.py
similarity index 67%
rename from examples/machine_translation/transformer/deploy/serving/utils/recorder.py
rename to legacy/examples/machine_translation/transformer/deploy/serving/utils/recorder.py
index 70a156a5e4f8..454abaf0e101 100644
--- a/examples/machine_translation/transformer/deploy/serving/utils/recorder.py
+++ b/legacy/examples/machine_translation/transformer/deploy/serving/utils/recorder.py
@@ -1,4 +1,19 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
import time
+
import paddle
diff --git a/examples/machine_translation/transformer/export_model.py b/legacy/examples/machine_translation/transformer/export_model.py
similarity index 100%
rename from examples/machine_translation/transformer/export_model.py
rename to legacy/examples/machine_translation/transformer/export_model.py
diff --git a/examples/machine_translation/transformer/fast_transformer/README.md b/legacy/examples/machine_translation/transformer/fast_transformer/README.md
similarity index 100%
rename from examples/machine_translation/transformer/fast_transformer/README.md
rename to legacy/examples/machine_translation/transformer/fast_transformer/README.md
diff --git a/examples/machine_translation/transformer/fast_transformer/encoder_decoding_predict.py b/legacy/examples/machine_translation/transformer/fast_transformer/encoder_decoding_predict.py
similarity index 100%
rename from examples/machine_translation/transformer/fast_transformer/encoder_decoding_predict.py
rename to legacy/examples/machine_translation/transformer/fast_transformer/encoder_decoding_predict.py
diff --git a/examples/machine_translation/transformer/fast_transformer/export_model.py b/legacy/examples/machine_translation/transformer/fast_transformer/export_model.py
similarity index 100%
rename from examples/machine_translation/transformer/fast_transformer/export_model.py
rename to legacy/examples/machine_translation/transformer/fast_transformer/export_model.py
diff --git a/examples/machine_translation/transformer/images/multi_head_attention.png b/legacy/examples/machine_translation/transformer/images/multi_head_attention.png
similarity index 100%
rename from examples/machine_translation/transformer/images/multi_head_attention.png
rename to legacy/examples/machine_translation/transformer/images/multi_head_attention.png
diff --git a/examples/machine_translation/transformer/images/transformer_network.png b/legacy/examples/machine_translation/transformer/images/transformer_network.png
similarity index 100%
rename from examples/machine_translation/transformer/images/transformer_network.png
rename to legacy/examples/machine_translation/transformer/images/transformer_network.png
diff --git a/examples/machine_translation/transformer/predict.py b/legacy/examples/machine_translation/transformer/predict.py
similarity index 100%
rename from examples/machine_translation/transformer/predict.py
rename to legacy/examples/machine_translation/transformer/predict.py
diff --git a/examples/machine_translation/transformer/reader.py b/legacy/examples/machine_translation/transformer/reader.py
similarity index 100%
rename from examples/machine_translation/transformer/reader.py
rename to legacy/examples/machine_translation/transformer/reader.py
diff --git a/examples/machine_translation/transformer/static/predict.py b/legacy/examples/machine_translation/transformer/static/predict.py
similarity index 100%
rename from examples/machine_translation/transformer/static/predict.py
rename to legacy/examples/machine_translation/transformer/static/predict.py
diff --git a/examples/machine_translation/transformer/static/train.py b/legacy/examples/machine_translation/transformer/static/train.py
similarity index 100%
rename from examples/machine_translation/transformer/static/train.py
rename to legacy/examples/machine_translation/transformer/static/train.py
diff --git a/legacy/examples/machine_translation/transformer/tls/distributed_utils.py b/legacy/examples/machine_translation/transformer/tls/distributed_utils.py
new file mode 100644
index 000000000000..67a9ae4c7cee
--- /dev/null
+++ b/legacy/examples/machine_translation/transformer/tls/distributed_utils.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+import paddle.distributed as dist
+
+
+def all_gather_tokens(data):
+ """Gathers num of tokens from all nodes.
+ `data` should be a tensor of num of tokens.
+ """
+ if dist.get_world_size() < 2:
+ return data
+ if not hasattr(all_gather_tokens, "_in_buffer") or all_gather_tokens._in_buffer is None:
+ all_gather_tokens._in_buffer = data
+ all_gather_tokens._out_buffers = []
+ in_buffer = all_gather_tokens._in_buffer
+ out_buffers = all_gather_tokens._out_buffers
+
+ dist.all_gather(out_buffers, in_buffer)
+
+ return paddle.add_n(out_buffers)
diff --git a/examples/machine_translation/transformer/tls/record.py b/legacy/examples/machine_translation/transformer/tls/record.py
similarity index 50%
rename from examples/machine_translation/transformer/tls/record.py
rename to legacy/examples/machine_translation/transformer/tls/record.py
index d1ddc738a528..a5a6dddc7139 100644
--- a/examples/machine_translation/transformer/tls/record.py
+++ b/legacy/examples/machine_translation/transformer/tls/record.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
class AverageStatistical(object):
def __init__(self):
self.reset()
diff --git a/examples/machine_translation/transformer/tls/to_static.py b/legacy/examples/machine_translation/transformer/tls/to_static.py
similarity index 100%
rename from examples/machine_translation/transformer/tls/to_static.py
rename to legacy/examples/machine_translation/transformer/tls/to_static.py
diff --git a/examples/machine_translation/transformer/train.py b/legacy/examples/machine_translation/transformer/train.py
similarity index 100%
rename from examples/machine_translation/transformer/train.py
rename to legacy/examples/machine_translation/transformer/train.py
diff --git a/examples/model_compression/minilmv2/README.md b/legacy/examples/model_compression/minilmv2/README.md
similarity index 100%
rename from examples/model_compression/minilmv2/README.md
rename to legacy/examples/model_compression/minilmv2/README.md
diff --git a/examples/model_compression/minilmv2/general_distill.py b/legacy/examples/model_compression/minilmv2/general_distill.py
similarity index 100%
rename from examples/model_compression/minilmv2/general_distill.py
rename to legacy/examples/model_compression/minilmv2/general_distill.py
diff --git a/examples/model_compression/minilmv2/run_clue.py b/legacy/examples/model_compression/minilmv2/run_clue.py
similarity index 100%
rename from examples/model_compression/minilmv2/run_clue.py
rename to legacy/examples/model_compression/minilmv2/run_clue.py
diff --git a/examples/model_compression/ofa/README.md b/legacy/examples/model_compression/ofa/README.md
similarity index 100%
rename from examples/model_compression/ofa/README.md
rename to legacy/examples/model_compression/ofa/README.md
diff --git a/examples/model_compression/ofa/export_model.py b/legacy/examples/model_compression/ofa/export_model.py
similarity index 100%
rename from examples/model_compression/ofa/export_model.py
rename to legacy/examples/model_compression/ofa/export_model.py
diff --git a/examples/model_compression/ofa/imgs/ofa_bert.jpg b/legacy/examples/model_compression/ofa/imgs/ofa_bert.jpg
similarity index 100%
rename from examples/model_compression/ofa/imgs/ofa_bert.jpg
rename to legacy/examples/model_compression/ofa/imgs/ofa_bert.jpg
diff --git a/examples/model_compression/ofa/run_glue_ofa.py b/legacy/examples/model_compression/ofa/run_glue_ofa.py
similarity index 100%
rename from examples/model_compression/ofa/run_glue_ofa.py
rename to legacy/examples/model_compression/ofa/run_glue_ofa.py
diff --git a/examples/model_compression/ofa/run_glue_ofa_depth.py b/legacy/examples/model_compression/ofa/run_glue_ofa_depth.py
similarity index 100%
rename from examples/model_compression/ofa/run_glue_ofa_depth.py
rename to legacy/examples/model_compression/ofa/run_glue_ofa_depth.py
diff --git a/examples/model_compression/pp-minilm/README.md b/legacy/examples/model_compression/pp-minilm/README.md
similarity index 100%
rename from examples/model_compression/pp-minilm/README.md
rename to legacy/examples/model_compression/pp-minilm/README.md
diff --git a/examples/model_compression/pp-minilm/data.py b/legacy/examples/model_compression/pp-minilm/data.py
similarity index 100%
rename from examples/model_compression/pp-minilm/data.py
rename to legacy/examples/model_compression/pp-minilm/data.py
diff --git a/examples/model_compression/pp-minilm/deploy/python/infer.py b/legacy/examples/model_compression/pp-minilm/deploy/python/infer.py
similarity index 100%
rename from examples/model_compression/pp-minilm/deploy/python/infer.py
rename to legacy/examples/model_compression/pp-minilm/deploy/python/infer.py
diff --git a/examples/model_compression/pp-minilm/deploy/python/infer_all.sh b/legacy/examples/model_compression/pp-minilm/deploy/python/infer_all.sh
similarity index 100%
rename from examples/model_compression/pp-minilm/deploy/python/infer_all.sh
rename to legacy/examples/model_compression/pp-minilm/deploy/python/infer_all.sh
diff --git a/examples/model_compression/pp-minilm/deploy/python/infer_perf.sh b/legacy/examples/model_compression/pp-minilm/deploy/python/infer_perf.sh
similarity index 100%
rename from examples/model_compression/pp-minilm/deploy/python/infer_perf.sh
rename to legacy/examples/model_compression/pp-minilm/deploy/python/infer_perf.sh
diff --git a/examples/model_compression/pp-minilm/deploy/serving/README.md b/legacy/examples/model_compression/pp-minilm/deploy/serving/README.md
similarity index 100%
rename from examples/model_compression/pp-minilm/deploy/serving/README.md
rename to legacy/examples/model_compression/pp-minilm/deploy/serving/README.md
diff --git a/examples/model_compression/pp-minilm/deploy/serving/config_nlp.yml b/legacy/examples/model_compression/pp-minilm/deploy/serving/config_nlp.yml
similarity index 100%
rename from examples/model_compression/pp-minilm/deploy/serving/config_nlp.yml
rename to legacy/examples/model_compression/pp-minilm/deploy/serving/config_nlp.yml
diff --git a/examples/model_compression/pp-minilm/deploy/serving/export_to_serving.py b/legacy/examples/model_compression/pp-minilm/deploy/serving/export_to_serving.py
similarity index 100%
rename from examples/model_compression/pp-minilm/deploy/serving/export_to_serving.py
rename to legacy/examples/model_compression/pp-minilm/deploy/serving/export_to_serving.py
diff --git a/examples/model_compression/pp-minilm/deploy/serving/rpc_client.py b/legacy/examples/model_compression/pp-minilm/deploy/serving/rpc_client.py
similarity index 100%
rename from examples/model_compression/pp-minilm/deploy/serving/rpc_client.py
rename to legacy/examples/model_compression/pp-minilm/deploy/serving/rpc_client.py
index c975d13265e7..442391f969b2 100644
--- a/examples/model_compression/pp-minilm/deploy/serving/rpc_client.py
+++ b/legacy/examples/model_compression/pp-minilm/deploy/serving/rpc_client.py
@@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from paddle_serving_server.pipeline import PipelineClient
import numpy as np
+from paddle_serving_server.pipeline import PipelineClient
client = PipelineClient()
client.connect(["127.0.0.1:8091"])
diff --git a/examples/model_compression/pp-minilm/deploy/serving/web_service.py b/legacy/examples/model_compression/pp-minilm/deploy/serving/web_service.py
similarity index 100%
rename from examples/model_compression/pp-minilm/deploy/serving/web_service.py
rename to legacy/examples/model_compression/pp-minilm/deploy/serving/web_service.py
diff --git a/examples/model_compression/pp-minilm/finetuning/export_model.py b/legacy/examples/model_compression/pp-minilm/finetuning/export_model.py
similarity index 100%
rename from examples/model_compression/pp-minilm/finetuning/export_model.py
rename to legacy/examples/model_compression/pp-minilm/finetuning/export_model.py
diff --git a/examples/model_compression/pp-minilm/finetuning/run_all_search.sh b/legacy/examples/model_compression/pp-minilm/finetuning/run_all_search.sh
similarity index 68%
rename from examples/model_compression/pp-minilm/finetuning/run_all_search.sh
rename to legacy/examples/model_compression/pp-minilm/finetuning/run_all_search.sh
index c09a288a2fad..39364d1f4733 100644
--- a/examples/model_compression/pp-minilm/finetuning/run_all_search.sh
+++ b/legacy/examples/model_compression/pp-minilm/finetuning/run_all_search.sh
@@ -1,3 +1,17 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
# $1 means GENERAL_DIR
mkdir -p $1/afqmc
mkdir -p $1/tnews
diff --git a/examples/model_compression/pp-minilm/finetuning/run_clue.py b/legacy/examples/model_compression/pp-minilm/finetuning/run_clue.py
similarity index 100%
rename from examples/model_compression/pp-minilm/finetuning/run_clue.py
rename to legacy/examples/model_compression/pp-minilm/finetuning/run_clue.py
diff --git a/examples/model_compression/pp-minilm/finetuning/run_clue.sh b/legacy/examples/model_compression/pp-minilm/finetuning/run_clue.sh
similarity index 50%
rename from examples/model_compression/pp-minilm/finetuning/run_clue.sh
rename to legacy/examples/model_compression/pp-minilm/finetuning/run_clue.sh
index de7f577ee6af..f8e5061e657d 100644
--- a/examples/model_compression/pp-minilm/finetuning/run_clue.sh
+++ b/legacy/examples/model_compression/pp-minilm/finetuning/run_clue.sh
@@ -1,3 +1,16 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
export TASK_NAME=$1
export LR=$2
diff --git a/examples/model_compression/pp-minilm/finetuning/run_one_search.sh b/legacy/examples/model_compression/pp-minilm/finetuning/run_one_search.sh
similarity index 68%
rename from examples/model_compression/pp-minilm/finetuning/run_one_search.sh
rename to legacy/examples/model_compression/pp-minilm/finetuning/run_one_search.sh
index fbb5261d2f31..c15fef531b9c 100644
--- a/examples/model_compression/pp-minilm/finetuning/run_one_search.sh
+++ b/legacy/examples/model_compression/pp-minilm/finetuning/run_one_search.sh
@@ -1,3 +1,17 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
OUTPUT_DIR=$1
TASK_NAME=$2
diff --git a/examples/model_compression/pp-minilm/general_distill/README.md b/legacy/examples/model_compression/pp-minilm/general_distill/README.md
similarity index 100%
rename from examples/model_compression/pp-minilm/general_distill/README.md
rename to legacy/examples/model_compression/pp-minilm/general_distill/README.md
diff --git a/examples/model_compression/pp-minilm/general_distill/general_distill.py b/legacy/examples/model_compression/pp-minilm/general_distill/general_distill.py
similarity index 100%
rename from examples/model_compression/pp-minilm/general_distill/general_distill.py
rename to legacy/examples/model_compression/pp-minilm/general_distill/general_distill.py
diff --git a/examples/model_compression/pp-minilm/general_distill/run.sh b/legacy/examples/model_compression/pp-minilm/general_distill/run.sh
similarity index 100%
rename from examples/model_compression/pp-minilm/general_distill/run.sh
rename to legacy/examples/model_compression/pp-minilm/general_distill/run.sh
diff --git a/examples/model_compression/pp-minilm/pp-minilm.png b/legacy/examples/model_compression/pp-minilm/pp-minilm.png
similarity index 100%
rename from examples/model_compression/pp-minilm/pp-minilm.png
rename to legacy/examples/model_compression/pp-minilm/pp-minilm.png
diff --git a/examples/model_compression/pp-minilm/pruning/export.sh b/legacy/examples/model_compression/pp-minilm/pruning/export.sh
similarity index 100%
rename from examples/model_compression/pp-minilm/pruning/export.sh
rename to legacy/examples/model_compression/pp-minilm/pruning/export.sh
diff --git a/examples/model_compression/pp-minilm/pruning/export_all.sh b/legacy/examples/model_compression/pp-minilm/pruning/export_all.sh
similarity index 100%
rename from examples/model_compression/pp-minilm/pruning/export_all.sh
rename to legacy/examples/model_compression/pp-minilm/pruning/export_all.sh
diff --git a/examples/model_compression/pp-minilm/pruning/export_model.py b/legacy/examples/model_compression/pp-minilm/pruning/export_model.py
similarity index 100%
rename from examples/model_compression/pp-minilm/pruning/export_model.py
rename to legacy/examples/model_compression/pp-minilm/pruning/export_model.py
diff --git a/examples/model_compression/pp-minilm/pruning/prune.py b/legacy/examples/model_compression/pp-minilm/pruning/prune.py
similarity index 100%
rename from examples/model_compression/pp-minilm/pruning/prune.py
rename to legacy/examples/model_compression/pp-minilm/pruning/prune.py
diff --git a/examples/model_compression/pp-minilm/pruning/prune.sh b/legacy/examples/model_compression/pp-minilm/pruning/prune.sh
similarity index 100%
rename from examples/model_compression/pp-minilm/pruning/prune.sh
rename to legacy/examples/model_compression/pp-minilm/pruning/prune.sh
diff --git a/examples/model_compression/pp-minilm/quantization/quant_all.sh b/legacy/examples/model_compression/pp-minilm/quantization/quant_all.sh
similarity index 100%
rename from examples/model_compression/pp-minilm/quantization/quant_all.sh
rename to legacy/examples/model_compression/pp-minilm/quantization/quant_all.sh
diff --git a/examples/model_compression/pp-minilm/quantization/quant_post.py b/legacy/examples/model_compression/pp-minilm/quantization/quant_post.py
similarity index 100%
rename from examples/model_compression/pp-minilm/quantization/quant_post.py
rename to legacy/examples/model_compression/pp-minilm/quantization/quant_post.py
diff --git a/examples/question_generation/README.md b/legacy/examples/question_generation/README.md
similarity index 100%
rename from examples/question_generation/README.md
rename to legacy/examples/question_generation/README.md
diff --git a/examples/question_generation/t5/README.md b/legacy/examples/question_generation/t5/README.md
similarity index 100%
rename from examples/question_generation/t5/README.md
rename to legacy/examples/question_generation/t5/README.md
diff --git a/examples/question_generation/t5/predict.py b/legacy/examples/question_generation/t5/predict.py
similarity index 100%
rename from examples/question_generation/t5/predict.py
rename to legacy/examples/question_generation/t5/predict.py
diff --git a/examples/question_generation/t5/requirements.txt b/legacy/examples/question_generation/t5/requirements.txt
similarity index 100%
rename from examples/question_generation/t5/requirements.txt
rename to legacy/examples/question_generation/t5/requirements.txt
diff --git a/examples/question_generation/t5/train.py b/legacy/examples/question_generation/t5/train.py
similarity index 100%
rename from examples/question_generation/t5/train.py
rename to legacy/examples/question_generation/t5/train.py
diff --git a/examples/question_generation/t5/utils.py b/legacy/examples/question_generation/t5/utils.py
similarity index 100%
rename from examples/question_generation/t5/utils.py
rename to legacy/examples/question_generation/t5/utils.py
diff --git a/examples/question_generation/unimo-text/README.md b/legacy/examples/question_generation/unimo-text/README.md
similarity index 100%
rename from examples/question_generation/unimo-text/README.md
rename to legacy/examples/question_generation/unimo-text/README.md
diff --git a/examples/question_generation/unimo-text/deploy/paddle_inference/README.md b/legacy/examples/question_generation/unimo-text/deploy/paddle_inference/README.md
similarity index 100%
rename from examples/question_generation/unimo-text/deploy/paddle_inference/README.md
rename to legacy/examples/question_generation/unimo-text/deploy/paddle_inference/README.md
diff --git a/examples/question_generation/unimo-text/deploy/paddle_inference/infer_utils.py b/legacy/examples/question_generation/unimo-text/deploy/paddle_inference/infer_utils.py
similarity index 100%
rename from examples/question_generation/unimo-text/deploy/paddle_inference/infer_utils.py
rename to legacy/examples/question_generation/unimo-text/deploy/paddle_inference/infer_utils.py
diff --git a/examples/question_generation/unimo-text/deploy/paddle_inference/inference.py b/legacy/examples/question_generation/unimo-text/deploy/paddle_inference/inference.py
similarity index 100%
rename from examples/question_generation/unimo-text/deploy/paddle_inference/inference.py
rename to legacy/examples/question_generation/unimo-text/deploy/paddle_inference/inference.py
diff --git a/examples/question_generation/unimo-text/deploy/paddle_serving/README.md b/legacy/examples/question_generation/unimo-text/deploy/paddle_serving/README.md
similarity index 100%
rename from examples/question_generation/unimo-text/deploy/paddle_serving/README.md
rename to legacy/examples/question_generation/unimo-text/deploy/paddle_serving/README.md
diff --git a/examples/question_generation/unimo-text/deploy/paddle_serving/config.yml b/legacy/examples/question_generation/unimo-text/deploy/paddle_serving/config.yml
similarity index 100%
rename from examples/question_generation/unimo-text/deploy/paddle_serving/config.yml
rename to legacy/examples/question_generation/unimo-text/deploy/paddle_serving/config.yml
diff --git a/examples/question_generation/unimo-text/deploy/paddle_serving/infer_utils.py b/legacy/examples/question_generation/unimo-text/deploy/paddle_serving/infer_utils.py
similarity index 100%
rename from examples/question_generation/unimo-text/deploy/paddle_serving/infer_utils.py
rename to legacy/examples/question_generation/unimo-text/deploy/paddle_serving/infer_utils.py
diff --git a/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_client.py b/legacy/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_client.py
similarity index 100%
rename from examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_client.py
rename to legacy/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_client.py
diff --git a/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_service.py b/legacy/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_service.py
similarity index 100%
rename from examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_service.py
rename to legacy/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_service.py
diff --git a/examples/question_generation/unimo-text/export_model.py b/legacy/examples/question_generation/unimo-text/export_model.py
similarity index 100%
rename from examples/question_generation/unimo-text/export_model.py
rename to legacy/examples/question_generation/unimo-text/export_model.py
diff --git a/examples/question_generation/unimo-text/gen_utils.py b/legacy/examples/question_generation/unimo-text/gen_utils.py
similarity index 99%
rename from examples/question_generation/unimo-text/gen_utils.py
rename to legacy/examples/question_generation/unimo-text/gen_utils.py
index ecc75584d89f..f7868500987e 100644
--- a/examples/question_generation/unimo-text/gen_utils.py
+++ b/legacy/examples/question_generation/unimo-text/gen_utils.py
@@ -16,10 +16,10 @@
from functools import partial
import numpy as np
-
import paddle
import paddle.distributed as dist
-from paddle.io import DataLoader, DistributedBatchSampler, BatchSampler
+from paddle.io import BatchSampler, DataLoader, DistributedBatchSampler
+
from paddlenlp.data import Pad
diff --git a/examples/question_generation/unimo-text/predict.py b/legacy/examples/question_generation/unimo-text/predict.py
similarity index 100%
rename from examples/question_generation/unimo-text/predict.py
rename to legacy/examples/question_generation/unimo-text/predict.py
diff --git a/examples/question_generation/unimo-text/requirements.txt b/legacy/examples/question_generation/unimo-text/requirements.txt
similarity index 100%
rename from examples/question_generation/unimo-text/requirements.txt
rename to legacy/examples/question_generation/unimo-text/requirements.txt
diff --git a/examples/question_generation/unimo-text/train.py b/legacy/examples/question_generation/unimo-text/train.py
similarity index 100%
rename from examples/question_generation/unimo-text/train.py
rename to legacy/examples/question_generation/unimo-text/train.py
diff --git a/examples/semantic_indexing/NQdataset.py b/legacy/examples/semantic_indexing/NQdataset.py
similarity index 98%
rename from examples/semantic_indexing/NQdataset.py
rename to legacy/examples/semantic_indexing/NQdataset.py
index 58efe8156ce1..ca1de6adc23f 100644
--- a/examples/semantic_indexing/NQdataset.py
+++ b/legacy/examples/semantic_indexing/NQdataset.py
@@ -86,7 +86,7 @@ def _read_json_data(self, dataPath):
def __getitem__(self, index):
json_sample_data = self.data[index]
r = BiEncoderSample()
- r.query = self._porcess_query(json_sample_data["question"])
+ r.query = self._process_query(json_sample_data["question"])
positive_ctxs = json_sample_data["positive_ctxs"]
@@ -106,7 +106,7 @@ def create_passage(ctx):
return r
- def _porcess_query(self, query):
+ def _process_query(self, query):
query = normalize_question(query)
if self.query_special_suffix and not query.endswith(self.query_special_suffix):
diff --git a/examples/semantic_indexing/README.md b/legacy/examples/semantic_indexing/README.md
similarity index 73%
rename from examples/semantic_indexing/README.md
rename to legacy/examples/semantic_indexing/README.md
index 9b37dd24b737..f411744ceb0f 100644
--- a/examples/semantic_indexing/README.md
+++ b/legacy/examples/semantic_indexing/README.md
@@ -7,21 +7,21 @@
我们基于 ERNIE1.0 热启,分别采用 [In-batch negatives](https://arxiv.org/abs/2004.04906) 策略和 HardestNeg 策略开源了 [batch_neg_v1.0](https://bj.bcebos.com/paddlenlp/models/semantic_index/batch_neg_v1.0.tar) 和 [hardest_neg_v1.0](https://bj.bcebos.com/paddlenlp/models/semantic_index/hardest_neg_v1.0.tar) 模型,相比 Baseline 模型效果有显著提升:
## 效果评估
-| 模型 | Recall@10 | Recall@50 |策略简要说明|
-| ------------ | ------------ | ------------ |--------- |
-| Baseline | 46.99 | 60.84 | 标准 pair-wise 训练范式,通过随机采样产生负样本|
-| [In-batch negatives](https://arxiv.org/abs/2004.04906) | 51.20(**+4.21**) | 67.24(**+6.4**) | 在 Batch 内同时使用 batch_size 个负样本进行训练|
-| HardestNeg | 50.22(**+3.23**) | 65.17(**+4.33**) | 在 Batch 内先挖掘最难负样本,然后进行 pair-wise 训练
|
+| 模型 | Recall@10 | Recall@50 | 策略简要说明 |
+|--------------------------------------------------------|------------------|------------------|---------------------------------------------------------------------------------------|
+| Baseline | 46.99 | 60.84 | 标准 pair-wise 训练范式,通过随机采样产生负样本 |
+| [In-batch negatives](https://arxiv.org/abs/2004.04906) | 51.20(**+4.21**) | 67.24(**+6.4**) | 在 Batch 内同时使用 batch_size 个负样本进行训练 |
+| HardestNeg | 50.22(**+3.23**) | 65.17(**+4.33**) | 在 Batch 内先挖掘最难负样本,然后进行 pair-wise 训练
|
## 语义索引预训练模型下载
以下模型结构参数为:
`TrasformerLayer:12, Hidden:768, Heads:12, OutputEmbSize: 256`
-|Model|训练参数配置|硬件|MD5|
-| ------------ | ------------ | ------------ |-----------|
-|[batch_neg_v1.0](https://bj.bcebos.com/paddlenlp/models/semantic_index/batch_neg_v1.0.tar)|margin:0.2 scale:30 epoch:3 lr:5E-5 bs:128 max_len:64
|单卡v100-16g
|da1bb1487bd3fd6a53b8ef95c278f3e6|
-|[hardest_neg_v1.0](https://bj.bcebos.com/paddlenlp/models/semantic_index/hardest_neg_v1.0.tar)|margin:0.2 epoch:3 lr:5E-5 bs:128 max_len:64 |单卡v100-16g|b535d890110ea608c8562c525a0b84b5|
+| Model | 训练参数配置 | 硬件 | MD5 |
+|------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------|----------------------------------------------|----------------------------------|
+| [batch_neg_v1.0](https://bj.bcebos.com/paddlenlp/models/semantic_index/batch_neg_v1.0.tar) | margin:0.2 scale:30 epoch:3 lr:5E-5 bs:128 max_len:64
| 单卡v100-16g
| da1bb1487bd3fd6a53b8ef95c278f3e6 |
+| [hardest_neg_v1.0](https://bj.bcebos.com/paddlenlp/models/semantic_index/hardest_neg_v1.0.tar) | margin:0.2 epoch:3 lr:5E-5 bs:128 max_len:64 | 单卡v100-16g | b535d890110ea608c8562c525a0b84b5 |
## 数据准备
@@ -47,11 +47,11 @@
### 数据下载
-|数据|描述|数量|MD5|
-| ------------ | ------------ | ------------ | -------- |
-|[训练集(semantic_pair_train.tsv)](https://bj.bcebos.com/paddlenlp/models/semantic_index/semantic_pair_train.tsv)
|每行为语义相似的文本 Pair 构成的训练集
|222546|590286f695200160350cc5838cb34f00|
-|[评估集(same_semantic.tsv)](https://bj.bcebos.com/paddlenlp/models/semantic_index/same_semantic.tsv)|每行为语义相似文本 Pair 构成的评估集|10255|86ec1fd5234d944177574372dcf780c5|
-|[召回库(corpus_file)](https://bj.bcebos.com/paddlenlp/models/semantic_index/corpus_file)|每行为单条文本构成的召回库|313714|a3fbc3421b5aeb939809876fc7beeaa8|
+| 数据 | 描述 | 数量 | MD5 |
+|--------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------|--------|----------------------------------|
+| [训练集(semantic_pair_train.tsv)](https://bj.bcebos.com/paddlenlp/models/semantic_index/semantic_pair_train.tsv)
| 每行为语义相似的文本 Pair 构成的训练集
| 222546 | 590286f695200160350cc5838cb34f00 |
+| [评估集(same_semantic.tsv)](https://bj.bcebos.com/paddlenlp/models/semantic_index/same_semantic.tsv) | 每行为语义相似文本 Pair 构成的评估集 | 10255 | 86ec1fd5234d944177574372dcf780c5 |
+| [召回库(corpus_file)](https://bj.bcebos.com/paddlenlp/models/semantic_index/corpus_file) | 每行为单条文本构成的召回库 | 313714 | a3fbc3421b5aeb939809876fc7beeaa8 |
## 项目依赖:
@@ -242,17 +242,17 @@ python -u -m paddle.distributed.launch --gpus "0" \
详细性能评测数据如下表:
-| batch size | max_seq_len | Paddle 前向(ms)|FT FP32(ms) | FT FP16(ms) |Speedup(FT FP32/Paddle)|Speedup(FT FP16/Paddle)|
-| ---------- | ----------- | ------------------- | ------------------- |------------------ |------------------ |------------------ |
-| 16 | 16 | 23.56 | 5.40 | 5.38 | 4.36| 4.38|
-| 16 | 32 | 22.34 | 8.11 | 5.57|2.75|4.01|
-| 16 | 64 | 22.79 | 14.84 |5.39|1.54|4.23|
-| 32 | 16 | 23.41 | 8.16 |5.30|2.87|4.42|
-| 32 | 32 | 22.67 | 14.84 |6.21|1.53|3.65|
-| 32 | 64 | 33.49 | 28.53 |6.05|1.17|5.54|
-| 64 | 16 | 22.60 | 14.81 |5.59|1.53|4.04|
-| 64 | 32 | 33.52 | 28.22 |6.24|1.19|5.37|
-| 64 | 64 | 62.62 | 55.25 |11.55|1.13|5.42|
+| batch size | max_seq_len | Paddle 前向(ms) | FT FP32(ms) | FT FP16(ms) | Speedup(FT FP32/Paddle) | Speedup(FT FP16/Paddle) |
+|------------|-------------|-----------------|-------------|-------------|-------------------------|-------------------------|
+| 16 | 16 | 23.56 | 5.40 | 5.38 | 4.36 | 4.38 |
+| 16 | 32 | 22.34 | 8.11 | 5.57 | 2.75 | 4.01 |
+| 16 | 64 | 22.79 | 14.84 | 5.39 | 1.54 | 4.23 |
+| 32 | 16 | 23.41 | 8.16 | 5.30 | 2.87 | 4.42 |
+| 32 | 32 | 22.67 | 14.84 | 6.21 | 1.53 | 3.65 |
+| 32 | 64 | 33.49 | 28.53 | 6.05 | 1.17 | 5.54 |
+| 64 | 16 | 22.60 | 14.81 | 5.59 | 1.53 | 4.04 |
+| 64 | 32 | 33.52 | 28.22 | 6.24 | 1.19 | 5.37 |
+| 64 | 64 | 62.62 | 55.25 | 11.55 | 1.13 | 5.42 |
Note: 测试环境如下
```
diff --git a/examples/semantic_indexing/README_gradient_cache.md b/legacy/examples/semantic_indexing/README_gradient_cache.md
similarity index 100%
rename from examples/semantic_indexing/README_gradient_cache.md
rename to legacy/examples/semantic_indexing/README_gradient_cache.md
diff --git a/examples/semantic_indexing/ance/model.py b/legacy/examples/semantic_indexing/ance/model.py
similarity index 100%
rename from examples/semantic_indexing/ance/model.py
rename to legacy/examples/semantic_indexing/ance/model.py
diff --git a/examples/semantic_indexing/ann_util.py b/legacy/examples/semantic_indexing/ann_util.py
similarity index 99%
rename from examples/semantic_indexing/ann_util.py
rename to legacy/examples/semantic_indexing/ann_util.py
index 55c608d3e58c..652d38c91010 100644
--- a/examples/semantic_indexing/ann_util.py
+++ b/legacy/examples/semantic_indexing/ann_util.py
@@ -14,8 +14,9 @@
# coding=UTF-8
-import numpy as np
import hnswlib
+import numpy as np
+
from paddlenlp.utils.log import logger
diff --git a/examples/semantic_indexing/base_model.py b/legacy/examples/semantic_indexing/base_model.py
similarity index 100%
rename from examples/semantic_indexing/base_model.py
rename to legacy/examples/semantic_indexing/base_model.py
diff --git a/examples/semantic_indexing/batch_negative/model.py b/legacy/examples/semantic_indexing/batch_negative/model.py
similarity index 94%
rename from examples/semantic_indexing/batch_negative/model.py
rename to legacy/examples/semantic_indexing/batch_negative/model.py
index fd87c6d8363e..a091f0d2d730 100644
--- a/examples/semantic_indexing/batch_negative/model.py
+++ b/legacy/examples/semantic_indexing/batch_negative/model.py
@@ -23,7 +23,7 @@ def __init__(self, pretrained_model, dropout=None, margin=0.3, scale=30, output_
self.margin = margin
# Used scaling cosine similarity to ease converge
- self.sacle = scale
+ self.scale = scale
def forward(
self,
@@ -47,7 +47,7 @@ def forward(
cosine_sim = paddle.matmul(query_cls_embedding, title_cls_embedding, transpose_y=True)
- # substract margin from all positive samples cosine_sim()
+ # subtract margin from all positive samples cosine_sim()
margin_diag = paddle.full(
shape=[query_cls_embedding.shape[0]], fill_value=self.margin, dtype=paddle.get_default_dtype()
)
@@ -55,7 +55,7 @@ def forward(
cosine_sim = cosine_sim - paddle.diag(margin_diag)
# scale cosine to ease training converge
- cosine_sim *= self.sacle
+ cosine_sim *= self.scale
labels = paddle.arange(0, query_cls_embedding.shape[0], dtype="int64")
labels = paddle.reshape(labels, shape=[-1, 1])
diff --git a/examples/semantic_indexing/biencoder_base_model.py b/legacy/examples/semantic_indexing/biencoder_base_model.py
similarity index 100%
rename from examples/semantic_indexing/biencoder_base_model.py
rename to legacy/examples/semantic_indexing/biencoder_base_model.py
diff --git a/examples/semantic_indexing/data.py b/legacy/examples/semantic_indexing/data.py
similarity index 94%
rename from examples/semantic_indexing/data.py
rename to legacy/examples/semantic_indexing/data.py
index c8e2e232f370..7bc340d3bcb0 100644
--- a/examples/semantic_indexing/data.py
+++ b/legacy/examples/semantic_indexing/data.py
@@ -84,7 +84,7 @@ def read_text_triplet(data_path):
# ANN - active learning ------------------------------------------------------
def get_latest_checkpoint(args):
"""
- Return: (latest_checkpint_path, global_step)
+ Return: (latest_checkpoint_path, global_step)
"""
if not os.path.exists(args.save_dir):
return args.init_from_ckpt, 0
@@ -114,7 +114,7 @@ def get_latest_ann_data(ann_data_dir):
def valid_checkpoint(step):
ann_data_file = os.path.join(ann_data_dir, step, "new_ann_data")
- # succed_flag_file is an empty file that indicates ann data has been generated
+ # succeed_flag_file is an empty file that indicates ann data has been generated
succeed_flag_file = os.path.join(ann_data_dir, step, "succeed_flag_file")
return os.path.exists(succeed_flag_file) and os.path.exists(ann_data_file)
@@ -122,7 +122,7 @@ def valid_checkpoint(step):
if len(ann_data_steps) > 0:
latest_ann_data_file = os.path.join(ann_data_dir, str(max(ann_data_steps)), "new_ann_data")
- logger.info("Using lateset ann_data_file:{}".format(latest_ann_data_file))
+ logger.info("Using latest ann_data_file:{}".format(latest_ann_data_file))
return latest_ann_data_file, max(ann_data_steps)
logger.info("no new ann_data, return (None, -1)")
@@ -142,8 +142,8 @@ def gen_text_file(similar_text_pair_file):
texts = []
with open(similar_text_pair_file, "r", encoding="utf-8") as f:
for line in f:
- splited_line = line.rstrip().split("\t")
- if len(splited_line) != 2:
+ splitted_line = line.rstrip().split("\t")
+ if len(splitted_line) != 2:
continue
text, similar_text = line.rstrip().split("\t")
diff --git a/examples/semantic_indexing/dense_retriever.py b/legacy/examples/semantic_indexing/dense_retriever.py
similarity index 100%
rename from examples/semantic_indexing/dense_retriever.py
rename to legacy/examples/semantic_indexing/dense_retriever.py
diff --git a/examples/semantic_indexing/evaluate.py b/legacy/examples/semantic_indexing/evaluate.py
similarity index 95%
rename from examples/semantic_indexing/evaluate.py
rename to legacy/examples/semantic_indexing/evaluate.py
index bfa086e521c9..2c1fc5a949df 100644
--- a/examples/semantic_indexing/evaluate.py
+++ b/legacy/examples/semantic_indexing/evaluate.py
@@ -21,7 +21,7 @@
"--similar_text_pair",
type=str,
default="",
- help="The full path of similat pair file",
+ help="The full path of similar pair file",
)
parser.add_argument(
"--recall_result_file",
@@ -33,7 +33,7 @@
"--recall_num",
type=int,
default=10,
- help="Most similair number of doc recalled from corpus per query",
+ help="Most similar number of doc recalled from corpus per query",
)
args = parser.parse_args()
diff --git a/examples/semantic_indexing/faiss_indexer.py b/legacy/examples/semantic_indexing/faiss_indexer.py
similarity index 97%
rename from examples/semantic_indexing/faiss_indexer.py
rename to legacy/examples/semantic_indexing/faiss_indexer.py
index a0a2eb9aa7db..3ab0b18f0c9a 100644
--- a/examples/semantic_indexing/faiss_indexer.py
+++ b/legacy/examples/semantic_indexing/faiss_indexer.py
@@ -20,14 +20,14 @@
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
"""
- FAISS-based index components for dense retriver
+ FAISS-based index components for dense retriever
"""
-import os
-import time
import logging
+import os
import pickle
-from typing import List, Tuple, Iterator
+import time
+from typing import Iterator, List, Tuple
import faiss
import numpy as np
@@ -143,7 +143,7 @@ def __init__(
super(DenseHNSWFlatIndexer, self).__init__(buffer_size=buffer_size)
# IndexHNSWFlat supports L2 similarity only
- # so we have to apply DOT -> L2 similairy space conversion with the help of an extra dimension
+ # so we have to apply DOT -> L2 similarity space conversion with the help of an extra dimension
index = faiss.IndexHNSWFlat(vector_sz + 1, store_n)
index.hnsw.efSearch = ef_search
index.hnsw.efConstruction = ef_construction
diff --git a/examples/semantic_indexing/fast_predict.py b/legacy/examples/semantic_indexing/fast_predict.py
similarity index 100%
rename from examples/semantic_indexing/fast_predict.py
rename to legacy/examples/semantic_indexing/fast_predict.py
diff --git a/examples/semantic_indexing/generate_dense_embeddings.py b/legacy/examples/semantic_indexing/generate_dense_embeddings.py
similarity index 100%
rename from examples/semantic_indexing/generate_dense_embeddings.py
rename to legacy/examples/semantic_indexing/generate_dense_embeddings.py
diff --git a/examples/semantic_indexing/gradient_cache/model.py b/legacy/examples/semantic_indexing/gradient_cache/model.py
similarity index 96%
rename from examples/semantic_indexing/gradient_cache/model.py
rename to legacy/examples/semantic_indexing/gradient_cache/model.py
index 04745d097889..9c5388c7665e 100644
--- a/examples/semantic_indexing/gradient_cache/model.py
+++ b/legacy/examples/semantic_indexing/gradient_cache/model.py
@@ -22,7 +22,7 @@ def __init__(self, pretrained_model, dropout=None, margin=0.3, scale=30, output_
super().__init__(pretrained_model, dropout, output_emb_size)
self.margin = margin
# Used scaling cosine similarity to ease converge
- self.sacle = scale
+ self.scale = scale
def get_pooled_embedding_with_no_grad(
self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None
@@ -77,7 +77,7 @@ def forward(
cosine_sim = paddle.matmul(query_cls_embedding, title_cls_embedding, transpose_y=True)
- # substract margin from all positive samples cosine_sim()
+ # subtract margin from all positive samples cosine_sim()
margin_diag = paddle.full(
shape=[query_cls_embedding.shape[0]], fill_value=self.margin, dtype=paddle.get_default_dtype()
)
@@ -85,7 +85,7 @@ def forward(
cosine_sim = cosine_sim - paddle.diag(margin_diag)
# scale cosine to ease training converge
- cosine_sim *= self.sacle
+ cosine_sim *= self.scale
labels = paddle.arange(0, query_cls_embedding.shape[0], dtype="int64")
labels = paddle.reshape(labels, shape=[-1, 1])
diff --git a/examples/semantic_indexing/hardest_negative/model.py b/legacy/examples/semantic_indexing/hardest_negative/model.py
similarity index 89%
rename from examples/semantic_indexing/hardest_negative/model.py
rename to legacy/examples/semantic_indexing/hardest_negative/model.py
index ce4db41341c2..3e7676e4214b 100644
--- a/examples/semantic_indexing/hardest_negative/model.py
+++ b/legacy/examples/semantic_indexing/hardest_negative/model.py
@@ -46,12 +46,12 @@ def forward(
pos_sim = paddle.max(cosine_sim, axis=-1)
- # subtract 10000 from all diagnal elements of cosine_sim
- mask_socre = paddle.full(
+ # subtract 10000 from all diagonal elements of cosine_sim
+ mask_score = paddle.full(
shape=[query_cls_embedding.shape[0]], fill_value=10000, dtype=paddle.get_default_dtype()
)
- tmp_cosin_sim = cosine_sim - paddle.diag(mask_socre)
- hardest_negative_sim = paddle.max(tmp_cosin_sim, axis=-1)
+ tmp_cosine_sim = cosine_sim - paddle.diag(mask_score)
+ hardest_negative_sim = paddle.max(tmp_cosine_sim, axis=-1)
labels = paddle.full(shape=[query_cls_embedding.shape[0]], fill_value=1.0, dtype="float32")
diff --git a/examples/semantic_indexing/predict.py b/legacy/examples/semantic_indexing/predict.py
similarity index 98%
rename from examples/semantic_indexing/predict.py
rename to legacy/examples/semantic_indexing/predict.py
index 741bb4ffdf45..0459c0120922 100644
--- a/examples/semantic_indexing/predict.py
+++ b/legacy/examples/semantic_indexing/predict.py
@@ -116,6 +116,6 @@ def predict(model, data_loader):
if args.use_fp16:
convert_to_fp16(model.ptm.encoder)
- cosin_sim = predict(model, valid_data_loader)
- for idx, cosine in enumerate(cosin_sim):
+ cosine_sim = predict(model, valid_data_loader)
+ for idx, cosine in enumerate(cosine_sim):
print("{}".format(cosine))
diff --git a/examples/semantic_indexing/qa_validation.py b/legacy/examples/semantic_indexing/qa_validation.py
similarity index 97%
rename from examples/semantic_indexing/qa_validation.py
rename to legacy/examples/semantic_indexing/qa_validation.py
index e4be203ec57a..499730673002 100644
--- a/examples/semantic_indexing/qa_validation.py
+++ b/legacy/examples/semantic_indexing/qa_validation.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
- Set of utilities for Q&A results validation tasks - Retriver passage validation and Reader predicted answer validation
+ Set of utilities for Q&A results validation tasks - Retriever passage validation and Reader predicted answer validation
"""
import collections
@@ -21,7 +21,8 @@
import unicodedata
from functools import partial
from multiprocessing import Pool as ProcessPool
-from typing import Tuple, List, Dict
+from typing import Dict, List, Tuple
+
import regex as re
from tokenizers import SimpleTokenizer
diff --git a/examples/semantic_indexing/recall.py b/legacy/examples/semantic_indexing/recall.py
similarity index 100%
rename from examples/semantic_indexing/recall.py
rename to legacy/examples/semantic_indexing/recall.py
diff --git a/examples/semantic_indexing/requirements.txt b/legacy/examples/semantic_indexing/requirements.txt
similarity index 100%
rename from examples/semantic_indexing/requirements.txt
rename to legacy/examples/semantic_indexing/requirements.txt
diff --git a/examples/semantic_indexing/run_ann_data_gen.py b/legacy/examples/semantic_indexing/run_ann_data_gen.py
similarity index 100%
rename from examples/semantic_indexing/run_ann_data_gen.py
rename to legacy/examples/semantic_indexing/run_ann_data_gen.py
diff --git a/examples/semantic_indexing/tokenizers.py b/legacy/examples/semantic_indexing/tokenizers.py
similarity index 100%
rename from examples/semantic_indexing/tokenizers.py
rename to legacy/examples/semantic_indexing/tokenizers.py
diff --git a/examples/semantic_indexing/train_ance.py b/legacy/examples/semantic_indexing/train_ance.py
similarity index 100%
rename from examples/semantic_indexing/train_ance.py
rename to legacy/examples/semantic_indexing/train_ance.py
diff --git a/examples/semantic_indexing/train_batch_neg.py b/legacy/examples/semantic_indexing/train_batch_neg.py
similarity index 100%
rename from examples/semantic_indexing/train_batch_neg.py
rename to legacy/examples/semantic_indexing/train_batch_neg.py
diff --git a/examples/semantic_indexing/train_gradient_cache.py b/legacy/examples/semantic_indexing/train_gradient_cache.py
similarity index 100%
rename from examples/semantic_indexing/train_gradient_cache.py
rename to legacy/examples/semantic_indexing/train_gradient_cache.py
diff --git a/examples/semantic_indexing/train_gradient_cache_DPR.py b/legacy/examples/semantic_indexing/train_gradient_cache_DPR.py
similarity index 100%
rename from examples/semantic_indexing/train_gradient_cache_DPR.py
rename to legacy/examples/semantic_indexing/train_gradient_cache_DPR.py
diff --git a/examples/semantic_indexing/train_hardest_neg.py b/legacy/examples/semantic_indexing/train_hardest_neg.py
similarity index 100%
rename from examples/semantic_indexing/train_hardest_neg.py
rename to legacy/examples/semantic_indexing/train_hardest_neg.py
diff --git a/examples/sentiment_analysis/skep/README.md b/legacy/examples/sentiment_analysis/skep/README.md
similarity index 100%
rename from examples/sentiment_analysis/skep/README.md
rename to legacy/examples/sentiment_analysis/skep/README.md
diff --git a/examples/sentiment_analysis/skep/deploy/python/predict.py b/legacy/examples/sentiment_analysis/skep/deploy/python/predict.py
similarity index 100%
rename from examples/sentiment_analysis/skep/deploy/python/predict.py
rename to legacy/examples/sentiment_analysis/skep/deploy/python/predict.py
diff --git a/examples/sentiment_analysis/skep/export_model.py b/legacy/examples/sentiment_analysis/skep/export_model.py
similarity index 100%
rename from examples/sentiment_analysis/skep/export_model.py
rename to legacy/examples/sentiment_analysis/skep/export_model.py
diff --git a/examples/sentiment_analysis/skep/predict_aspect.py b/legacy/examples/sentiment_analysis/skep/predict_aspect.py
similarity index 100%
rename from examples/sentiment_analysis/skep/predict_aspect.py
rename to legacy/examples/sentiment_analysis/skep/predict_aspect.py
diff --git a/examples/sentiment_analysis/skep/predict_opinion.py b/legacy/examples/sentiment_analysis/skep/predict_opinion.py
similarity index 100%
rename from examples/sentiment_analysis/skep/predict_opinion.py
rename to legacy/examples/sentiment_analysis/skep/predict_opinion.py
diff --git a/examples/sentiment_analysis/skep/predict_sentence.py b/legacy/examples/sentiment_analysis/skep/predict_sentence.py
similarity index 100%
rename from examples/sentiment_analysis/skep/predict_sentence.py
rename to legacy/examples/sentiment_analysis/skep/predict_sentence.py
diff --git a/examples/sentiment_analysis/skep/train_aspect.py b/legacy/examples/sentiment_analysis/skep/train_aspect.py
similarity index 100%
rename from examples/sentiment_analysis/skep/train_aspect.py
rename to legacy/examples/sentiment_analysis/skep/train_aspect.py
diff --git a/examples/sentiment_analysis/skep/train_opinion.py b/legacy/examples/sentiment_analysis/skep/train_opinion.py
similarity index 100%
rename from examples/sentiment_analysis/skep/train_opinion.py
rename to legacy/examples/sentiment_analysis/skep/train_opinion.py
diff --git a/examples/sentiment_analysis/skep/train_sentence.py b/legacy/examples/sentiment_analysis/skep/train_sentence.py
similarity index 100%
rename from examples/sentiment_analysis/skep/train_sentence.py
rename to legacy/examples/sentiment_analysis/skep/train_sentence.py
diff --git a/examples/simultaneous_translation/stacl/README.md b/legacy/examples/simultaneous_translation/stacl/README.md
similarity index 91%
rename from examples/simultaneous_translation/stacl/README.md
rename to legacy/examples/simultaneous_translation/stacl/README.md
index 554b251a4b61..1bd391caa538 100644
--- a/examples/simultaneous_translation/stacl/README.md
+++ b/legacy/examples/simultaneous_translation/stacl/README.md
@@ -133,13 +133,13 @@ perl mosesdecoder/scripts/generic/multi-bleu.perl newstest2017.tok.en < predict.
## 模型下载(更新中)
我们提供基于NIST(中->英,共2M中英句对)预训练模型,供大家下载,下载后需解压使用。
-| Wait-k策略 | 模型连接 | 4-ref BLEU on NIST 2008|
-| ------------ | --------------- |---------|
-| Wait-1 | [下载](https://bj.bcebos.com/paddlenlp/models/stacl/nist_zhen_full_w1.tar.gz) |30.94|
-| Wait-3 |[下载](https://bj.bcebos.com/paddlenlp/models/stacl/nist_zhen_full_w3.tar.gz) |34.24 |
-| Wait-5 |[下载](https://bj.bcebos.com/paddlenlp/models/stacl/nist_zhen_full_w5.tar.gz) |36.30 |
-| Wait-7 |[下载](https://bj.bcebos.com/paddlenlp/models/stacl/nist_zhen_full_w7.tar.gz) |37.84 |
-| Wait_-1(整句模型) |[下载](https://bj.bcebos.com/paddlenlp/models/stacl/nist_zhen_full_sent.tar.gz) |41.41 |
+| Wait-k策略 | 模型连接 | 4-ref BLEU on NIST 2008 |
+|-------------------|---------------------------------------------------------------------------------|-------------------------|
+| Wait-1 | [下载](https://bj.bcebos.com/paddlenlp/models/stacl/nist_zhen_full_w1.tar.gz) | 30.94 |
+| Wait-3 | [下载](https://bj.bcebos.com/paddlenlp/models/stacl/nist_zhen_full_w3.tar.gz) | 34.24 |
+| Wait-5 | [下载](https://bj.bcebos.com/paddlenlp/models/stacl/nist_zhen_full_w5.tar.gz) | 36.30 |
+| Wait-7 | [下载](https://bj.bcebos.com/paddlenlp/models/stacl/nist_zhen_full_w7.tar.gz) | 37.84 |
+| Wait_-1(整句模型) | [下载](https://bj.bcebos.com/paddlenlp/models/stacl/nist_zhen_full_sent.tar.gz) | 41.41 |
词表下载:[source vocab](https://bj.bcebos.com/paddlenlp/models/stacl/nist.20k.zh.vocab) ,[target vocab](https://bj.bcebos.com/paddlenlp/models/stacl/nist.10k.en.vocab)
## Demo展示
diff --git a/examples/simultaneous_translation/stacl/config/transformer.yaml b/legacy/examples/simultaneous_translation/stacl/config/transformer.yaml
similarity index 100%
rename from examples/simultaneous_translation/stacl/config/transformer.yaml
rename to legacy/examples/simultaneous_translation/stacl/config/transformer.yaml
diff --git a/examples/simultaneous_translation/stacl/demo/README.md b/legacy/examples/simultaneous_translation/stacl/demo/README.md
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/README.md
rename to legacy/examples/simultaneous_translation/stacl/demo/README.md
diff --git a/examples/simultaneous_translation/stacl/demo/README_ai.md b/legacy/examples/simultaneous_translation/stacl/demo/README_ai.md
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/README_ai.md
rename to legacy/examples/simultaneous_translation/stacl/demo/README_ai.md
diff --git a/examples/simultaneous_translation/stacl/demo/const.py b/legacy/examples/simultaneous_translation/stacl/demo/const.py
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/const.py
rename to legacy/examples/simultaneous_translation/stacl/demo/const.py
diff --git a/examples/simultaneous_translation/stacl/demo/demo.py b/legacy/examples/simultaneous_translation/stacl/demo/demo.py
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/demo.py
rename to legacy/examples/simultaneous_translation/stacl/demo/demo.py
diff --git a/examples/simultaneous_translation/stacl/demo/images/paddlenlp.png b/legacy/examples/simultaneous_translation/stacl/demo/images/paddlenlp.png
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/images/paddlenlp.png
rename to legacy/examples/simultaneous_translation/stacl/demo/images/paddlenlp.png
diff --git a/examples/simultaneous_translation/stacl/demo/images/speech_demo_show.gif b/legacy/examples/simultaneous_translation/stacl/demo/images/speech_demo_show.gif
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/images/speech_demo_show.gif
rename to legacy/examples/simultaneous_translation/stacl/demo/images/speech_demo_show.gif
diff --git a/examples/simultaneous_translation/stacl/demo/images/step1.png b/legacy/examples/simultaneous_translation/stacl/demo/images/step1.png
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/images/step1.png
rename to legacy/examples/simultaneous_translation/stacl/demo/images/step1.png
diff --git a/examples/simultaneous_translation/stacl/demo/images/step2.png b/legacy/examples/simultaneous_translation/stacl/demo/images/step2.png
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/images/step2.png
rename to legacy/examples/simultaneous_translation/stacl/demo/images/step2.png
diff --git a/examples/simultaneous_translation/stacl/demo/images/step3.png b/legacy/examples/simultaneous_translation/stacl/demo/images/step3.png
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/images/step3.png
rename to legacy/examples/simultaneous_translation/stacl/demo/images/step3.png
diff --git a/examples/simultaneous_translation/stacl/demo/images/step4.png b/legacy/examples/simultaneous_translation/stacl/demo/images/step4.png
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/images/step4.png
rename to legacy/examples/simultaneous_translation/stacl/demo/images/step4.png
diff --git a/examples/simultaneous_translation/stacl/demo/images/step5.png b/legacy/examples/simultaneous_translation/stacl/demo/images/step5.png
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/images/step5.png
rename to legacy/examples/simultaneous_translation/stacl/demo/images/step5.png
diff --git a/examples/simultaneous_translation/stacl/demo/images/step6.png b/legacy/examples/simultaneous_translation/stacl/demo/images/step6.png
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/images/step6.png
rename to legacy/examples/simultaneous_translation/stacl/demo/images/step6.png
diff --git a/examples/simultaneous_translation/stacl/demo/images/step7.png b/legacy/examples/simultaneous_translation/stacl/demo/images/step7.png
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/images/step7.png
rename to legacy/examples/simultaneous_translation/stacl/demo/images/step7.png
diff --git a/examples/simultaneous_translation/stacl/demo/images/step8.png b/legacy/examples/simultaneous_translation/stacl/demo/images/step8.png
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/images/step8.png
rename to legacy/examples/simultaneous_translation/stacl/demo/images/step8.png
diff --git a/examples/simultaneous_translation/stacl/demo/images/text_demo_show.gif b/legacy/examples/simultaneous_translation/stacl/demo/images/text_demo_show.gif
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/images/text_demo_show.gif
rename to legacy/examples/simultaneous_translation/stacl/demo/images/text_demo_show.gif
diff --git a/examples/simultaneous_translation/stacl/demo/model_demo.py b/legacy/examples/simultaneous_translation/stacl/demo/model_demo.py
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/model_demo.py
rename to legacy/examples/simultaneous_translation/stacl/demo/model_demo.py
diff --git a/examples/simultaneous_translation/stacl/demo/requirements.txt b/legacy/examples/simultaneous_translation/stacl/demo/requirements.txt
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/requirements.txt
rename to legacy/examples/simultaneous_translation/stacl/demo/requirements.txt
diff --git a/examples/simultaneous_translation/stacl/demo/transformer_demo.yaml b/legacy/examples/simultaneous_translation/stacl/demo/transformer_demo.yaml
similarity index 100%
rename from examples/simultaneous_translation/stacl/demo/transformer_demo.yaml
rename to legacy/examples/simultaneous_translation/stacl/demo/transformer_demo.yaml
diff --git a/examples/simultaneous_translation/stacl/images/STACL_architecture.png b/legacy/examples/simultaneous_translation/stacl/images/STACL_architecture.png
similarity index 100%
rename from examples/simultaneous_translation/stacl/images/STACL_architecture.png
rename to legacy/examples/simultaneous_translation/stacl/images/STACL_architecture.png
diff --git a/examples/simultaneous_translation/stacl/images/example.png b/legacy/examples/simultaneous_translation/stacl/images/example.png
similarity index 100%
rename from examples/simultaneous_translation/stacl/images/example.png
rename to legacy/examples/simultaneous_translation/stacl/images/example.png
diff --git a/examples/simultaneous_translation/stacl/model.py b/legacy/examples/simultaneous_translation/stacl/model.py
similarity index 100%
rename from examples/simultaneous_translation/stacl/model.py
rename to legacy/examples/simultaneous_translation/stacl/model.py
diff --git a/examples/simultaneous_translation/stacl/predict.py b/legacy/examples/simultaneous_translation/stacl/predict.py
similarity index 99%
rename from examples/simultaneous_translation/stacl/predict.py
rename to legacy/examples/simultaneous_translation/stacl/predict.py
index 8f2e3da9e404..e2a6f7256158 100644
--- a/examples/simultaneous_translation/stacl/predict.py
+++ b/legacy/examples/simultaneous_translation/stacl/predict.py
@@ -12,17 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import os
import argparse
+import os
from pprint import pprint
-import yaml
-from attrdict import AttrDict
import paddle
-from paddlenlp.transformers import position_encoding_init
import reader
+import yaml
+from attrdict import AttrDict
from model import SimultaneousTransformer
+from paddlenlp.transformers import position_encoding_init
+
def parse_args():
parser = argparse.ArgumentParser()
diff --git a/examples/simultaneous_translation/stacl/reader.py b/legacy/examples/simultaneous_translation/stacl/reader.py
similarity index 99%
rename from examples/simultaneous_translation/stacl/reader.py
rename to legacy/examples/simultaneous_translation/stacl/reader.py
index cb71b2bfb212..cb0ab155d8ba 100644
--- a/examples/simultaneous_translation/stacl/reader.py
+++ b/legacy/examples/simultaneous_translation/stacl/reader.py
@@ -13,8 +13,10 @@
# limitations under the License.
from functools import partial
+
from paddle.io import DataLoader
-from paddlenlp.data import Vocab, Pad
+
+from paddlenlp.data import Pad, Vocab
from paddlenlp.data.sampler import SamplerHelper
from paddlenlp.datasets import load_dataset
diff --git a/examples/simultaneous_translation/stacl/requirements.txt b/legacy/examples/simultaneous_translation/stacl/requirements.txt
similarity index 100%
rename from examples/simultaneous_translation/stacl/requirements.txt
rename to legacy/examples/simultaneous_translation/stacl/requirements.txt
diff --git a/examples/simultaneous_translation/stacl/train.py b/legacy/examples/simultaneous_translation/stacl/train.py
similarity index 99%
rename from examples/simultaneous_translation/stacl/train.py
rename to legacy/examples/simultaneous_translation/stacl/train.py
index 09ecb03001a9..6fd3a80fda71 100644
--- a/examples/simultaneous_translation/stacl/train.py
+++ b/legacy/examples/simultaneous_translation/stacl/train.py
@@ -12,23 +12,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import argparse
import os
import time
-
-import argparse
from pprint import pprint
-import numpy as np
-import yaml
-from attrdict import AttrDict
+import numpy as np
import paddle
import paddle.distributed as dist
-from paddlenlp.utils.log import logger
-
import reader
-from model import SimultaneousTransformer, CrossEntropyCriterion
+import yaml
+from attrdict import AttrDict
+from model import CrossEntropyCriterion, SimultaneousTransformer
from utils.record import AverageStatistical
+from paddlenlp.utils.log import logger
+
def parse_args():
parser = argparse.ArgumentParser()
diff --git a/examples/text_graph/erniesage/models/__init__.py b/legacy/examples/simultaneous_translation/stacl/utils/__init__.py
similarity index 80%
rename from examples/text_graph/erniesage/models/__init__.py
rename to legacy/examples/simultaneous_translation/stacl/utils/__init__.py
index 4b02ff01793b..fd05a9208165 100644
--- a/examples/text_graph/erniesage/models/__init__.py
+++ b/legacy/examples/simultaneous_translation/stacl/utils/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,8 +11,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-from models import model
-
-__all__ = []
-__all__ += model.__all__
diff --git a/examples/simultaneous_translation/stacl/utils/record.py b/legacy/examples/simultaneous_translation/stacl/utils/record.py
similarity index 100%
rename from examples/simultaneous_translation/stacl/utils/record.py
rename to legacy/examples/simultaneous_translation/stacl/utils/record.py
diff --git a/examples/simultaneous_translation/stacl/utils/tokenizer.py b/legacy/examples/simultaneous_translation/stacl/utils/tokenizer.py
similarity index 100%
rename from examples/simultaneous_translation/stacl/utils/tokenizer.py
rename to legacy/examples/simultaneous_translation/stacl/utils/tokenizer.py
diff --git a/examples/torch_migration/README.md b/legacy/examples/torch_migration/README.md
similarity index 100%
rename from examples/torch_migration/README.md
rename to legacy/examples/torch_migration/README.md
diff --git a/examples/torch_migration/docs/ThesisReproduction_NLP.md b/legacy/examples/torch_migration/docs/ThesisReproduction_NLP.md
similarity index 100%
rename from examples/torch_migration/docs/ThesisReproduction_NLP.md
rename to legacy/examples/torch_migration/docs/ThesisReproduction_NLP.md
diff --git a/examples/torch_migration/pipeline/Step1/README.md b/legacy/examples/torch_migration/pipeline/Step1/README.md
similarity index 100%
rename from examples/torch_migration/pipeline/Step1/README.md
rename to legacy/examples/torch_migration/pipeline/Step1/README.md
diff --git a/examples/torch_migration/pipeline/Step1/check_step1.py b/legacy/examples/torch_migration/pipeline/Step1/check_step1.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step1/check_step1.py
rename to legacy/examples/torch_migration/pipeline/Step1/check_step1.py
diff --git a/examples/torch_migration/pipeline/Step1/pd_forward_bert.py b/legacy/examples/torch_migration/pipeline/Step1/pd_forward_bert.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step1/pd_forward_bert.py
rename to legacy/examples/torch_migration/pipeline/Step1/pd_forward_bert.py
diff --git a/examples/torch_migration/pipeline/Step1/pt_forward_bert.py b/legacy/examples/torch_migration/pipeline/Step1/pt_forward_bert.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step1/pt_forward_bert.py
rename to legacy/examples/torch_migration/pipeline/Step1/pt_forward_bert.py
diff --git a/examples/torch_migration/pipeline/Step1/torch2paddle.py b/legacy/examples/torch_migration/pipeline/Step1/torch2paddle.py
similarity index 99%
rename from examples/torch_migration/pipeline/Step1/torch2paddle.py
rename to legacy/examples/torch_migration/pipeline/Step1/torch2paddle.py
index 4a2b4977051b..b395486e83eb 100644
--- a/examples/torch_migration/pipeline/Step1/torch2paddle.py
+++ b/legacy/examples/torch_migration/pipeline/Step1/torch2paddle.py
@@ -17,9 +17,10 @@
import numpy as np
import paddle
import torch
-from paddlenlp.transformers import BertForPretraining as PDBertForMaskedLM
from transformers import BertForMaskedLM as PTBertForMaskedLM
+from paddlenlp.transformers import BertForPretraining as PDBertForMaskedLM
+
def convert_pytorch_checkpoint_to_paddle(
pytorch_checkpoint_path="pytorch_model.bin",
diff --git a/examples/torch_migration/pipeline/Step2/README.md b/legacy/examples/torch_migration/pipeline/Step2/README.md
similarity index 100%
rename from examples/torch_migration/pipeline/Step2/README.md
rename to legacy/examples/torch_migration/pipeline/Step2/README.md
diff --git a/examples/torch_migration/pipeline/Step2/accuracy.py b/legacy/examples/torch_migration/pipeline/Step2/accuracy.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step2/accuracy.py
rename to legacy/examples/torch_migration/pipeline/Step2/accuracy.py
diff --git a/examples/torch_migration/pipeline/Step2/check_step2.py b/legacy/examples/torch_migration/pipeline/Step2/check_step2.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step2/check_step2.py
rename to legacy/examples/torch_migration/pipeline/Step2/check_step2.py
diff --git a/examples/torch_migration/pipeline/Step2/demo_sst2_sentence/demo.tsv b/legacy/examples/torch_migration/pipeline/Step2/demo_sst2_sentence/demo.tsv
similarity index 100%
rename from examples/torch_migration/pipeline/Step2/demo_sst2_sentence/demo.tsv
rename to legacy/examples/torch_migration/pipeline/Step2/demo_sst2_sentence/demo.tsv
diff --git a/examples/torch_migration/pipeline/Step2/predict.py b/legacy/examples/torch_migration/pipeline/Step2/predict.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step2/predict.py
rename to legacy/examples/torch_migration/pipeline/Step2/predict.py
diff --git a/examples/torch_migration/pipeline/Step2/test_data.py b/legacy/examples/torch_migration/pipeline/Step2/test_data.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step2/test_data.py
rename to legacy/examples/torch_migration/pipeline/Step2/test_data.py
diff --git a/examples/torch_migration/pipeline/Step2/test_metric.py b/legacy/examples/torch_migration/pipeline/Step2/test_metric.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step2/test_metric.py
rename to legacy/examples/torch_migration/pipeline/Step2/test_metric.py
diff --git a/examples/torch_migration/pipeline/Step3/README.md b/legacy/examples/torch_migration/pipeline/Step3/README.md
similarity index 100%
rename from examples/torch_migration/pipeline/Step3/README.md
rename to legacy/examples/torch_migration/pipeline/Step3/README.md
diff --git a/examples/torch_migration/pipeline/Step3/check_step3.py b/legacy/examples/torch_migration/pipeline/Step3/check_step3.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step3/check_step3.py
rename to legacy/examples/torch_migration/pipeline/Step3/check_step3.py
diff --git a/examples/torch_migration/pipeline/Step3/paddle_loss.py b/legacy/examples/torch_migration/pipeline/Step3/paddle_loss.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step3/paddle_loss.py
rename to legacy/examples/torch_migration/pipeline/Step3/paddle_loss.py
diff --git a/examples/torch_migration/pipeline/Step3/torch_loss.py b/legacy/examples/torch_migration/pipeline/Step3/torch_loss.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step3/torch_loss.py
rename to legacy/examples/torch_migration/pipeline/Step3/torch_loss.py
diff --git a/examples/torch_migration/pipeline/Step4/README.md b/legacy/examples/torch_migration/pipeline/Step4/README.md
similarity index 100%
rename from examples/torch_migration/pipeline/Step4/README.md
rename to legacy/examples/torch_migration/pipeline/Step4/README.md
diff --git a/examples/torch_migration/pipeline/Step4/check_step4.py b/legacy/examples/torch_migration/pipeline/Step4/check_step4.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step4/check_step4.py
rename to legacy/examples/torch_migration/pipeline/Step4/check_step4.py
diff --git a/examples/torch_migration/pipeline/Step4/test_bp.py b/legacy/examples/torch_migration/pipeline/Step4/test_bp.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step4/test_bp.py
rename to legacy/examples/torch_migration/pipeline/Step4/test_bp.py
diff --git a/examples/torch_migration/pipeline/Step4/test_lr_scheduler.py b/legacy/examples/torch_migration/pipeline/Step4/test_lr_scheduler.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step4/test_lr_scheduler.py
rename to legacy/examples/torch_migration/pipeline/Step4/test_lr_scheduler.py
diff --git a/examples/torch_migration/pipeline/Step5/README.md b/legacy/examples/torch_migration/pipeline/Step5/README.md
similarity index 100%
rename from examples/torch_migration/pipeline/Step5/README.md
rename to legacy/examples/torch_migration/pipeline/Step5/README.md
diff --git a/examples/torch_migration/pipeline/Step5/bert_paddle/train.py b/legacy/examples/torch_migration/pipeline/Step5/bert_paddle/train.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step5/bert_paddle/train.py
rename to legacy/examples/torch_migration/pipeline/Step5/bert_paddle/train.py
diff --git a/examples/torch_migration/pipeline/Step5/bert_paddle/train.sh b/legacy/examples/torch_migration/pipeline/Step5/bert_paddle/train.sh
similarity index 100%
rename from examples/torch_migration/pipeline/Step5/bert_paddle/train.sh
rename to legacy/examples/torch_migration/pipeline/Step5/bert_paddle/train.sh
diff --git a/examples/torch_migration/pipeline/Step5/bert_paddle/utils.py b/legacy/examples/torch_migration/pipeline/Step5/bert_paddle/utils.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step5/bert_paddle/utils.py
rename to legacy/examples/torch_migration/pipeline/Step5/bert_paddle/utils.py
diff --git a/examples/torch_migration/pipeline/Step5/bert_torch/accuracy.py b/legacy/examples/torch_migration/pipeline/Step5/bert_torch/accuracy.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step5/bert_torch/accuracy.py
rename to legacy/examples/torch_migration/pipeline/Step5/bert_torch/accuracy.py
diff --git a/examples/torch_migration/pipeline/Step5/bert_torch/glue.py b/legacy/examples/torch_migration/pipeline/Step5/bert_torch/glue.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step5/bert_torch/glue.py
rename to legacy/examples/torch_migration/pipeline/Step5/bert_torch/glue.py
diff --git a/examples/torch_migration/pipeline/Step5/bert_torch/train.py b/legacy/examples/torch_migration/pipeline/Step5/bert_torch/train.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step5/bert_torch/train.py
rename to legacy/examples/torch_migration/pipeline/Step5/bert_torch/train.py
diff --git a/examples/torch_migration/pipeline/Step5/bert_torch/train.sh b/legacy/examples/torch_migration/pipeline/Step5/bert_torch/train.sh
similarity index 100%
rename from examples/torch_migration/pipeline/Step5/bert_torch/train.sh
rename to legacy/examples/torch_migration/pipeline/Step5/bert_torch/train.sh
diff --git a/examples/torch_migration/pipeline/Step5/bert_torch/utils.py b/legacy/examples/torch_migration/pipeline/Step5/bert_torch/utils.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step5/bert_torch/utils.py
rename to legacy/examples/torch_migration/pipeline/Step5/bert_torch/utils.py
diff --git a/examples/torch_migration/pipeline/Step5/check_step5.py b/legacy/examples/torch_migration/pipeline/Step5/check_step5.py
similarity index 100%
rename from examples/torch_migration/pipeline/Step5/check_step5.py
rename to legacy/examples/torch_migration/pipeline/Step5/check_step5.py
diff --git a/examples/torch_migration/pipeline/classifier_weights/generate_classifier_weights.py b/legacy/examples/torch_migration/pipeline/classifier_weights/generate_classifier_weights.py
similarity index 100%
rename from examples/torch_migration/pipeline/classifier_weights/generate_classifier_weights.py
rename to legacy/examples/torch_migration/pipeline/classifier_weights/generate_classifier_weights.py
diff --git a/examples/torch_migration/pipeline/fake_data/gen_fake_data.py b/legacy/examples/torch_migration/pipeline/fake_data/gen_fake_data.py
similarity index 100%
rename from examples/torch_migration/pipeline/fake_data/gen_fake_data.py
rename to legacy/examples/torch_migration/pipeline/fake_data/gen_fake_data.py
diff --git a/examples/torch_migration/pipeline/models/pd_bert.py b/legacy/examples/torch_migration/pipeline/models/pd_bert.py
similarity index 100%
rename from examples/torch_migration/pipeline/models/pd_bert.py
rename to legacy/examples/torch_migration/pipeline/models/pd_bert.py
diff --git a/examples/torch_migration/pipeline/models/pt_bert.py b/legacy/examples/torch_migration/pipeline/models/pt_bert.py
similarity index 100%
rename from examples/torch_migration/pipeline/models/pt_bert.py
rename to legacy/examples/torch_migration/pipeline/models/pt_bert.py
diff --git a/examples/torch_migration/pipeline/reprod_log_demo/check_log_diff.py b/legacy/examples/torch_migration/pipeline/reprod_log_demo/check_log_diff.py
similarity index 100%
rename from examples/torch_migration/pipeline/reprod_log_demo/check_log_diff.py
rename to legacy/examples/torch_migration/pipeline/reprod_log_demo/check_log_diff.py
diff --git a/examples/torch_migration/pipeline/reprod_log_demo/write_log.py b/legacy/examples/torch_migration/pipeline/reprod_log_demo/write_log.py
similarity index 100%
rename from examples/torch_migration/pipeline/reprod_log_demo/write_log.py
rename to legacy/examples/torch_migration/pipeline/reprod_log_demo/write_log.py
diff --git a/examples/torch_migration/pipeline/weights/torch2paddle.py b/legacy/examples/torch_migration/pipeline/weights/torch2paddle.py
similarity index 99%
rename from examples/torch_migration/pipeline/weights/torch2paddle.py
rename to legacy/examples/torch_migration/pipeline/weights/torch2paddle.py
index 74511fea26e9..3a8d472064bd 100644
--- a/examples/torch_migration/pipeline/weights/torch2paddle.py
+++ b/legacy/examples/torch_migration/pipeline/weights/torch2paddle.py
@@ -17,9 +17,10 @@
import numpy as np
import paddle
import torch
-from paddlenlp.transformers import BertForPretraining as PDBertForMaskedLM
from transformers import BertForMaskedLM as PTBertForMaskedLM
+from paddlenlp.transformers import BertForPretraining as PDBertForMaskedLM
+
def convert_pytorch_checkpoint_to_paddle(
pytorch_checkpoint_path="pytorch_model.bin",
diff --git a/examples/torch_migration/pipeline/weights/torch_bert_weight.py b/legacy/examples/torch_migration/pipeline/weights/torch_bert_weight.py
similarity index 100%
rename from examples/torch_migration/pipeline/weights/torch_bert_weight.py
rename to legacy/examples/torch_migration/pipeline/weights/torch_bert_weight.py
index 819229e156a5..b1cf6f1881c3 100644
--- a/examples/torch_migration/pipeline/weights/torch_bert_weight.py
+++ b/legacy/examples/torch_migration/pipeline/weights/torch_bert_weight.py
@@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from transformers import BertModel
import torch
+from transformers import BertModel
hf_model = BertModel.from_pretrained("bert-base-uncased")
hf_model.eval()
diff --git a/examples/torch_migration/requirements.txt b/legacy/examples/torch_migration/requirements.txt
similarity index 100%
rename from examples/torch_migration/requirements.txt
rename to legacy/examples/torch_migration/requirements.txt
diff --git a/scripts/regression/ci_case.sh b/scripts/regression/ci_case.sh
index e19a42f8a756..44f41d61d212 100644
--- a/scripts/regression/ci_case.sh
+++ b/scripts/regression/ci_case.sh
@@ -22,412 +22,396 @@ export CXX_COMPILER_PATH=$(which g++)
export CC=$(which gcc)
export CXX=$(which g++)
-if [ ! -d "model_logs" ];then
+if [ ! -d "model_logs" ]; then
mkdir model_logs
fi
-if [ ! -d "unittest_logs" ];then
+if [ ! -d "unittest_logs" ]; then
mkdir model_logs
fi
-print_info(){
-if [ $1 -ne 0 ];then
- if [[ $2 =~ 'tests' ]];then
- mv ${nlp_dir}/unittest_logs/$3.log ${nlp_dir}/unittest_logs/$3_FAIL.log
- echo -e "\033[31m ${nlp_dir}/unittest_logs/$3_FAIL \033[0m"
- cat ${nlp_dir}/unittest_logs/$3_FAIL.log
+print_info() {
+ if [ $1 -ne 0 ]; then
+ if [[ $2 =~ 'tests' ]]; then
+ mv ${nlp_dir}/unittest_logs/$3.log ${nlp_dir}/unittest_logs/$3_FAIL.log
+ echo -e "\033[31m ${nlp_dir}/unittest_logs/$3_FAIL \033[0m"
+ cat ${nlp_dir}/unittest_logs/$3_FAIL.log
+ else
+ mv ${log_path}/$2 ${log_path}/$2_FAIL.log
+ echo -e "\033[31m ${log_path}/$2_FAIL \033[0m"
+ cat ${log_path}/$2_FAIL.log
+ fi
+ elif [[ $2 =~ 'tests' ]]; then
+ echo -e "\033[32m ${log_path}/$3_SUCCESS \033[0m"
else
- mv ${log_path}/$2 ${log_path}/$2_FAIL.log
- echo -e "\033[31m ${log_path}/$2_FAIL \033[0m"
- cat ${log_path}/$2_FAIL.log
+ echo -e "\033[32m ${log_path}/$2_SUCCESS \033[0m"
fi
-elif [[ $2 =~ 'tests' ]];then
- echo -e "\033[32m ${log_path}/$3_SUCCESS \033[0m"
-else
- echo -e "\033[32m ${log_path}/$2_SUCCESS \033[0m"
-fi
}
# case list
-# 1 waybill_ie (无可控参数,数据集外置)
-waybill_ie(){
-cd ${nlp_dir}/examples/information_extraction/waybill_ie/
-export CUDA_VISIBLE_DEVICES=${cudaid1}
-# BiGRU +CRF star training
-time (
-python download.py --data_dir ./waybill_ie
-python run_bigru_crf.py >${log_path}/waybill_ie_bigru_crf) >>${log_path}/waybill_ie_bigru_crf 2>&1
-print_info $? waybill_ie_bigru_crf
-# ERNIE +RF star training
-time (python run_ernie.py >${log_path}/waybill_ie_ernie) >>${log_path}/waybill_ie_ernie 2>&1
-print_info $? waybill_ie_ernie
-# ERNIE +CRF star training
-time (python run_ernie_crf.py >${log_path}/waybill_ie_ernie_crf) >>${log_path}/waybill_ie_ernie_crf 2>&1
-print_info $? waybill_ie_ernie_crf
-}
# 2 msra_ner (不可控,内置)
-msra_ner(){
-cd ${nlp_dir}/examples/information_extraction/msra_ner/
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-## train
-time (python -m paddle.distributed.launch ./train.py \
- --model_type bert \
- --model_name_or_path bert-base-multilingual-uncased \
- --dataset msra_ner \
- --max_seq_length 128 \
- --batch_size 16 \
- --learning_rate 2e-5 \
- --num_train_epochs 1 \
- --logging_steps 1 \
- --max_steps 2 \
- --save_steps 2 \
- --output_dir ./tmp/msra_ner/ \
- --device gpu >${log_path}/msra_ner_train) >>${log_path}/msra_ner_train 2>&1
-print_info $? msra_ner_train
-## eval
-time (python -u ./eval.py \
- --model_name_or_path bert-base-multilingual-uncased \
- --max_seq_length 128 \
- --batch_size 16 \
- --device gpu \
- --init_checkpoint_path ./tmp/msra_ner/model_2.pdparams >${log_path}/msra_ner_eval) >>${log_path}/msra_ner_eval 2>&1
-print_info $? msra_ner_eval
-## predict
-time (python -u ./predict.py \
- --model_name_or_path bert-base-multilingual-uncased \
- --max_seq_length 128 \
- --batch_size 16 \
- --device gpu \
- --init_checkpoint_path ./tmp/msra_ner/model_2.pdparams >${log_path}/msra_ner_predict) >>${log_path}/msra_ner_predict 2>&1
-print_info $? msra_ner_predict
+msra_ner() {
+ cd ${nlp_dir}/legacy/examples/information_extraction/msra_ner/
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ ## train
+ time (python -m paddle.distributed.launch ./train.py \
+ --model_type bert \
+ --model_name_or_path bert-base-multilingual-uncased \
+ --dataset msra_ner \
+ --max_seq_length 128 \
+ --batch_size 16 \
+ --learning_rate 2e-5 \
+ --num_train_epochs 1 \
+ --logging_steps 1 \
+ --max_steps 2 \
+ --save_steps 2 \
+ --output_dir ./tmp/msra_ner/ \
+ --device gpu >${log_path}/msra_ner_train) >>${log_path}/msra_ner_train 2>&1
+ print_info $? msra_ner_train
+ ## eval
+ time (python -u ./eval.py \
+ --model_name_or_path bert-base-multilingual-uncased \
+ --max_seq_length 128 \
+ --batch_size 16 \
+ --device gpu \
+ --init_checkpoint_path ./tmp/msra_ner/model_2.pdparams >${log_path}/msra_ner_eval) >>${log_path}/msra_ner_eval 2>&1
+ print_info $? msra_ner_eval
+ ## predict
+ time (python -u ./predict.py \
+ --model_name_or_path bert-base-multilingual-uncased \
+ --max_seq_length 128 \
+ --batch_size 16 \
+ --device gpu \
+ --init_checkpoint_path ./tmp/msra_ner/model_2.pdparams >${log_path}/msra_ner_predict) >>${log_path}/msra_ner_predict 2>&1
+ print_info $? msra_ner_predict
}
# 3 glue
glue() {
-cd ${nlp_dir}/examples/benchmark/glue/
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-## TASK_SST-2
-export TASK_NAME=SST-2
-time (python -u run_glue.py \
- --model_type bert \
- --model_name_or_path bert-base-uncased \
- --task_name $TASK_NAME \
- --max_seq_length 128 \
- --batch_size 128 \
- --learning_rate 3e-5 \
- --max_steps 1 \
- --logging_steps 1 \
- --save_steps 1 \
- --output_dir ./$TASK_NAME/ \
- --device gpu >${log_path}/glue_${TASK_NAME}_train) >>${log_path}/glue_${TASK_NAME}_train 2>&1
-print_info $? glue_${TASK_NAME}_train
+ cd ${nlp_dir}/legacy/examples/benchmark/glue/
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ ## TASK_SST-2
+ export TASK_NAME=SST-2
+ time (python -u run_glue.py \
+ --model_type bert \
+ --model_name_or_path bert-base-uncased \
+ --task_name $TASK_NAME \
+ --max_seq_length 128 \
+ --batch_size 128 \
+ --learning_rate 3e-5 \
+ --max_steps 1 \
+ --logging_steps 1 \
+ --save_steps 1 \
+ --output_dir ./$TASK_NAME/ \
+ --device gpu >${log_path}/glue_${TASK_NAME}_train) >>${log_path}/glue_${TASK_NAME}_train 2>&1
+ print_info $? glue_${TASK_NAME}_train
}
# 4 bert
bert() {
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-# cd ${nlp_dir}/model_zoo/bert/
-# wget -q https://paddle-qa.bj.bcebos.com/paddlenlp/bert.tar.gz
-# tar -xzvf bert.tar.gz
-python -c "import datasets;from datasets import load_dataset; train_dataset=load_dataset('glue', 'sst2', split='train')"
-cd ${nlp_dir}/model_zoo/bert/data/
-wget -q https://bj.bcebos.com/paddlenlp/models/transformers/bert/data/training_data.hdf5
-cd ../
-# pretrain
-time (python -m paddle.distributed.launch run_pretrain.py \
- --model_type bert \
- --model_name_or_path bert-base-uncased \
- --max_predictions_per_seq 20 \
- --batch_size 16 \
- --learning_rate 1e-4 \
- --weight_decay 1e-2 \
- --adam_epsilon 1e-6 \
- --warmup_steps 10000 \
- --input_dir data/ \
- --output_dir pretrained_models/ \
- --logging_steps 1 \
- --save_steps 1 \
- --max_steps 1 \
- --device gpu \
- --use_amp False >${log_path}/bert_pretrain) >>${log_path}/bert_pretrain 2>&1
-print_info $? bert_pretrain
-time (python -m paddle.distributed.launch run_glue_trainer.py \
- --model_name_or_path bert-base-uncased \
- --task_name SST2 \
- --max_seq_length 128 \
- --per_device_train_batch_size 32 \
- --per_device_eval_batch_size 32 \
- --learning_rate 2e-5 \
- --num_train_epochs 3 \
- --logging_steps 1 \
- --save_steps 1 \
- --max_steps 1 \
- --output_dir ./tmp/ \
- --device gpu \
- --fp16 False\
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ # cd ${nlp_dir}/model_zoo/bert/
+ # wget -q https://paddle-qa.bj.bcebos.com/paddlenlp/bert.tar.gz
+ # tar -xzvf bert.tar.gz
+ python -c "import datasets;from datasets import load_dataset; train_dataset=load_dataset('glue', 'sst2', split='train')"
+ cd ${nlp_dir}/model_zoo/bert/data/
+ wget -q https://bj.bcebos.com/paddlenlp/models/transformers/bert/data/training_data.hdf5
+ cd ../
+ # pretrain
+ time (python -m paddle.distributed.launch run_pretrain.py \
+ --model_type bert \
+ --model_name_or_path bert-base-uncased \
+ --max_predictions_per_seq 20 \
+ --batch_size 16 \
+ --learning_rate 1e-4 \
+ --weight_decay 1e-2 \
+ --adam_epsilon 1e-6 \
+ --warmup_steps 10000 \
+ --input_dir data/ \
+ --output_dir pretrained_models/ \
+ --logging_steps 1 \
+ --save_steps 1 \
+ --max_steps 1 \
+ --device gpu \
+ --use_amp False >${log_path}/bert_pretrain) >>${log_path}/bert_pretrain 2>&1
+ print_info $? bert_pretrain
+ time (python -m paddle.distributed.launch run_glue_trainer.py \
+ --model_name_or_path bert-base-uncased \
+ --task_name SST2 \
+ --max_seq_length 128 \
+ --per_device_train_batch_size 32 \
+ --per_device_eval_batch_size 32 \
+ --learning_rate 2e-5 \
+ --num_train_epochs 3 \
+ --logging_steps 1 \
+ --save_steps 1 \
+ --max_steps 1 \
+ --output_dir ./tmp/ \
+ --device gpu \
+ --fp16 False\
--do_train \
- --do_eval >${log_path}/bert_fintune) >>${log_path}/bert_fintune 2>&1
-print_info $? bert_fintune
-time (python -u ./export_model.py \
- --model_type bert \
- --model_path bert-base-uncased \
- --output_path ./infer_model/model >${log_path}/bert_export) >>${log_path}/bert_export 2>&1
-print_info $? bert_export
- }
+ --do_eval >${log_path}/bert_fintune) >>${log_path}/bert_fintune 2>&1
+ print_info $? bert_fintune
+ time (python -u ./export_model.py \
+ --model_type bert \
+ --model_path bert-base-uncased \
+ --output_path ./infer_model/model >${log_path}/bert_export) >>${log_path}/bert_export 2>&1
+ print_info $? bert_export
+}
# 5 skep (max save 不可控 内置)
-skep () {
-cd ${nlp_dir}/examples/sentiment_analysis/skep/
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-## train_sentence
-time ( python -m paddle.distributed.launch train_sentence.py --batch_size 16 --epochs 1 --model_name "skep_ernie_1.0_large_ch" --device gpu --save_dir ./checkpoints >${log_path}/skep_train_sentence) >>${log_path}/skep_train_sentence 2>&1
-print_info $? skep_train_sentence
-## train_aspect
-time ( python -m paddle.distributed.launch train_aspect.py --batch_size 4 --epochs 1 --device gpu --save_dir ./aspect_checkpoints >${log_path}/skep_train_aspect) >>${log_path}/skep_train_aspect 2>&1
-print_info $? skep_train_aspect
-# # train_opinion
-time ( python -m paddle.distributed.launch train_opinion.py --batch_size 4 --epochs 1 --device gpu --save_dir ./opinion_checkpoints >${log_path}/skep_train_opinion) >>${log_path}/skep_train_opinion 2>&1
-print_info $? skep_train_opinion
-# predict_sentence
-time (python predict_sentence.py --model_name "skep_ernie_1.0_large_ch" --ckpt_dir checkpoints/model_100 >${log_path}/skep_predict_sentence) >>${log_path}/skep_predict_sentence 2>&1
-print_info $? skep_predict_sentence
-## predict_aspect
-time (python predict_aspect.py --device 'gpu' --ckpt_dir ./aspect_checkpoints/model_100 >${log_path}/skep_predict_aspect) >>${log_path}/skep_predict_aspect 2>&1
-print_info $? skep_predict_aspect
-# # predict_opinion
-time (python predict_opinion.py --device 'gpu' --ckpt_dir ./opinion_checkpoints/model_100 >${log_path}/skep_predict_opinion) >>${log_path}/skep_predict_opinion 2>&1
-print_info $? skep_predict_opinion
+skep() {
+ cd ${nlp_dir}/legacy/examples/sentiment_analysis/skep/
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ ## train_sentence
+ time (python -m paddle.distributed.launch train_sentence.py --batch_size 16 --epochs 1 --model_name "skep_ernie_1.0_large_ch" --device gpu --save_dir ./checkpoints >${log_path}/skep_train_sentence) >>${log_path}/skep_train_sentence 2>&1
+ print_info $? skep_train_sentence
+ ## train_aspect
+ time (python -m paddle.distributed.launch train_aspect.py --batch_size 4 --epochs 1 --device gpu --save_dir ./aspect_checkpoints >${log_path}/skep_train_aspect) >>${log_path}/skep_train_aspect 2>&1
+ print_info $? skep_train_aspect
+ # # train_opinion
+ time (python -m paddle.distributed.launch train_opinion.py --batch_size 4 --epochs 1 --device gpu --save_dir ./opinion_checkpoints >${log_path}/skep_train_opinion) >>${log_path}/skep_train_opinion 2>&1
+ print_info $? skep_train_opinion
+ # predict_sentence
+ time (python predict_sentence.py --model_name "skep_ernie_1.0_large_ch" --ckpt_dir checkpoints/model_100 >${log_path}/skep_predict_sentence) >>${log_path}/skep_predict_sentence 2>&1
+ print_info $? skep_predict_sentence
+ ## predict_aspect
+ time (python predict_aspect.py --device 'gpu' --ckpt_dir ./aspect_checkpoints/model_100 >${log_path}/skep_predict_aspect) >>${log_path}/skep_predict_aspect 2>&1
+ print_info $? skep_predict_aspect
+ # # predict_opinion
+ time (python predict_opinion.py --device 'gpu' --ckpt_dir ./opinion_checkpoints/model_100 >${log_path}/skep_predict_opinion) >>${log_path}/skep_predict_opinion 2>&1
+ print_info $? skep_predict_opinion
}
# 6 bigbird
bigbird(){
-cd ${nlp_dir}/examples/language_model/bigbird/
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-time (python -m paddle.distributed.launch --log_dir log run_pretrain.py --model_name_or_path bigbird-base-uncased \
- --input_dir "./data" \
- --output_dir "output" \
- --batch_size 4 \
- --weight_decay 0.01 \
- --learning_rate 1e-5 \
- --max_steps 1 \
- --save_steps 1 \
- --logging_steps 1 \
- --max_encoder_length 512 \
- --max_pred_length 75 >${log_path}/bigbird_pretrain) >>${log_path}/bigbird_pretrain 2>&1
+ cd ${nlp_dir}/examples/language_model/bigbird/
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ time (python -m paddle.distributed.launch --log_dir log run_pretrain.py --model_name_or_path bigbird-base-uncased \
+ --input_dir "./data" \
+ --output_dir "output" \
+ --batch_size 4 \
+ --weight_decay 0.01 \
+ --learning_rate 1e-5 \
+ --max_steps 1 \
+ --save_steps 1 \
+ --logging_steps 1 \
+ --max_encoder_length 512 \
+ --max_pred_length 75 >${log_path}/bigbird_pretrain) >>${log_path}/bigbird_pretrain 2>&1
print_info $? bigbird_pretrain
}
# 7 electra
electra(){
-cd ${nlp_dir}/model_zoo/electra/
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-export DATA_DIR=./BookCorpus/
-wget -q https://paddle-qa.bj.bcebos.com/paddlenlp/BookCorpus.tar.gz && tar -xzvf BookCorpus.tar.gz
-time (python -u ./run_pretrain.py \
- --model_type electra \
- --model_name_or_path electra-small \
- --input_dir ./BookCorpus/ \
- --output_dir ./pretrain_model/ \
- --train_batch_size 64 \
- --learning_rate 5e-4 \
- --max_seq_length 128 \
- --weight_decay 1e-2 \
- --adam_epsilon 1e-6 \
- --warmup_steps 10000 \
- --num_train_epochs 4 \
- --logging_steps 1 \
- --save_steps 1 \
- --max_steps 1 \
- --device gpu >${log_path}/electra_pretrain) >>${log_path}/electra_pretrain 2>&1
-print_info $? electra_pretrain
+ cd ${nlp_dir}/model_zoo/electra/
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ export DATA_DIR=./BookCorpus/
+ wget -q https://paddle-qa.bj.bcebos.com/paddlenlp/BookCorpus.tar.gz && tar -xzvf BookCorpus.tar.gz
+ time (python -u ./run_pretrain.py \
+ --model_type electra \
+ --model_name_or_path electra-small \
+ --input_dir ./BookCorpus/ \
+ --output_dir ./pretrain_model/ \
+ --train_batch_size 64 \
+ --learning_rate 5e-4 \
+ --max_seq_length 128 \
+ --weight_decay 1e-2 \
+ --adam_epsilon 1e-6 \
+ --warmup_steps 10000 \
+ --num_train_epochs 4 \
+ --logging_steps 1 \
+ --save_steps 1 \
+ --max_steps 1 \
+ --device gpu >${log_path}/electra_pretrain) >>${log_path}/electra_pretrain 2>&1
+ print_info $? electra_pretrain
}
# 9 ernie
ernie(){
-#data process
-cd ${nlp_dir}/model_zoo/ernie-1.0/
-mkdir data
-cd ./data
-wget -q https://paddlenlp.bj.bcebos.com/models/transformers/data_tools/ernie_wudao_0903_92M_ids.npy
-wget -q https://paddlenlp.bj.bcebos.com/models/transformers/data_tools/ernie_wudao_0903_92M_idx.npz
-cd ../
-mkdir data_ernie_3.0 && cd data_ernie_3.0
-wget https://bj.bcebos.com/paddlenlp/models/transformers/data_tools/wudao_200g_sample_ernie-3.0-base-zh_ids.npy
-wget https://bj.bcebos.com/paddlenlp/models/transformers/data_tools/wudao_200g_sample_ernie-3.0-base-zh_idx.npz
-cd ../
-# pretrain_trainer
-python -u -m paddle.distributed.launch \
- --log_dir "output/trainer_log" \
- run_pretrain_trainer.py \
- --model_type "ernie" \
- --model_name_or_path "ernie-3.0-base-zh" \
- --tokenizer_name_or_path "ernie-3.0-base-zh" \
- --input_dir "./data_ernie_3.0" \
- --output_dir "output/trainer_log" \
- --split 949,50,1 \
- --max_seq_length 512 \
- --per_device_train_batch_size 16 \
- --per_device_eval_batch_size 32 \
- --fp16 \
- --fp16_opt_level "O2" \
- --learning_rate 0.0001 \
- --min_learning_rate 0.00001 \
- --max_steps 2 \
- --save_steps 2 \
- --weight_decay 0.01 \
- --warmup_ratio 0.01 \
- --max_grad_norm 1.0 \
- --logging_steps 1\
+ #data process
+ cd ${nlp_dir}/model_zoo/ernie-1.0/
+ mkdir data
+ cd ./data
+ wget -q https://paddlenlp.bj.bcebos.com/models/transformers/data_tools/ernie_wudao_0903_92M_ids.npy
+ wget -q https://paddlenlp.bj.bcebos.com/models/transformers/data_tools/ernie_wudao_0903_92M_idx.npz
+ cd ../
+ mkdir data_ernie_3.0 && cd data_ernie_3.0
+ wget https://bj.bcebos.com/paddlenlp/models/transformers/data_tools/wudao_200g_sample_ernie-3.0-base-zh_ids.npy
+ wget https://bj.bcebos.com/paddlenlp/models/transformers/data_tools/wudao_200g_sample_ernie-3.0-base-zh_idx.npz
+ cd ../
+ # pretrain_trainer
+ python -u -m paddle.distributed.launch \
+ --log_dir "output/trainer_log" \
+ run_pretrain_trainer.py \
+ --model_type "ernie" \
+ --model_name_or_path "ernie-3.0-base-zh" \
+ --tokenizer_name_or_path "ernie-3.0-base-zh" \
+ --input_dir "./data_ernie_3.0" \
+ --output_dir "output/trainer_log" \
+ --split 949,50,1 \
+ --max_seq_length 512 \
+ --per_device_train_batch_size 16 \
+ --per_device_eval_batch_size 32 \
+ --fp16 \
+ --fp16_opt_level "O2" \
+ --learning_rate 0.0001 \
+ --min_learning_rate 0.00001 \
+ --max_steps 2 \
+ --save_steps 2 \
+ --weight_decay 0.01 \
+ --warmup_ratio 0.01 \
+ --max_grad_norm 1.0 \
+ --logging_steps 1\
--dataloader_num_workers 4 \
- --eval_steps 1000 \
- --report_to "visualdl" \
- --disable_tqdm true \
- --do_train \
- --device "gpu" >${log_path}/ernie_1.0_pretrain_trainer >>${log_path}/ernie_1.0_pretrain_trainer 2>&1
+ --eval_steps 1000 \
+ --report_to "visualdl" \
+ --disable_tqdm true \
+ --do_train \
+ --device "gpu" >${log_path}/ernie_1.0_pretrain_trainer >>${log_path}/ernie_1.0_pretrain_trainer 2>&1
print_info $? ernie_1.0_pretrain_trainer
-# pretrain_static
-python -u -m paddle.distributed.launch \
- --log_dir "./log" \
- run_pretrain_static.py \
- --model_type "ernie" \
- --model_name_or_path "ernie-1.0-base-zh" \
- --tokenizer_name_or_path "ernie-1.0-base-zh" \
- --input_dir "./data/" \
- --output_dir "./output/" \
- --max_seq_len 512 \
- --micro_batch_size 16 \
- --global_batch_size 32 \
- --sharding_degree 1 \
- --dp_degree 2 \
- --use_sharding false \
- --use_amp true \
- --use_recompute false \
- --max_lr 0.0001 \
- --min_lr 0.00001 \
- --max_steps 4 \
- --save_steps 2 \
- --checkpoint_steps 5000 \
- --decay_steps 3960000 \
- --weight_decay 0.01 \
- --warmup_rate 0.0025 \
- --grad_clip 1.0 \
- --logging_freq 2\
+ # pretrain_static
+ python -u -m paddle.distributed.launch \
+ --log_dir "./log" \
+ run_pretrain_static.py \
+ --model_type "ernie" \
+ --model_name_or_path "ernie-1.0-base-zh" \
+ --tokenizer_name_or_path "ernie-1.0-base-zh" \
+ --input_dir "./data/" \
+ --output_dir "./output/" \
+ --max_seq_len 512 \
+ --micro_batch_size 16 \
+ --global_batch_size 32 \
+ --sharding_degree 1 \
+ --dp_degree 2 \
+ --use_sharding false \
+ --use_amp true \
+ --use_recompute false \
+ --max_lr 0.0001 \
+ --min_lr 0.00001 \
+ --max_steps 4 \
+ --save_steps 2 \
+ --checkpoint_steps 5000 \
+ --decay_steps 3960000 \
+ --weight_decay 0.01 \
+ --warmup_rate 0.0025 \
+ --grad_clip 1.0 \
+ --logging_freq 2\
--num_workers 2 \
- --eval_freq 1000 \
- --device "gpu" >${log_path}/ernie_1.0_pretrain_static >>${log_path}/ernie_1.0_pretrain_static 2>&1
+ --eval_freq 1000 \
+ --device "gpu" >${log_path}/ernie_1.0_pretrain_static >>${log_path}/ernie_1.0_pretrain_static 2>&1
print_info $? ernie_1.0_pretrain_static
}
# 10 xlnet
xlnet(){
-cd ${nlp_dir}/examples/language_model/xlnet/
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-time (python -m paddle.distributed.launch ./run_glue.py \
- --model_name_or_path xlnet-base-cased \
- --task_name SST-2 \
- --max_seq_length 128 \
- --batch_size 32 \
- --learning_rate 2e-5 \
- --num_train_epochs 3 \
- --max_steps 1 \
- --logging_steps 1 \
- --save_steps 1 \
- --output_dir ./xlnet/ >${log_path}/xlnet_train) >>${log_path}/xlnet_train 2>&1
-print_info $? xlnet_train
+ cd ${nlp_dir}/examples/language_model/xlnet/
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ time (python -m paddle.distributed.launch ./run_glue.py \
+ --model_name_or_path xlnet-base-cased \
+ --task_name SST-2 \
+ --max_seq_length 128 \
+ --batch_size 32 \
+ --learning_rate 2e-5 \
+ --num_train_epochs 3 \
+ --max_steps 1 \
+ --logging_steps 1 \
+ --save_steps 1 \
+ --output_dir ./xlnet/ >${log_path}/xlnet_train) >>${log_path}/xlnet_train 2>&1
+ print_info $? xlnet_train
}
# 11 ofa
ofa(){
-cd ${nlp_dir}/examples/model_compression/ofa/
-cd ../../benchmark/glue/
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-# finetuing
-time (python -u run_glue.py \
- --model_type bert \
- --model_name_or_path bert-base-uncased \
- --task_name SST-2 \
- --max_seq_length 128 \
- --batch_size 32 \
- --learning_rate 2e-5 \
- --num_train_epochs 1 \
- --max_steps 1 \
- --logging_steps 1 \
- --save_steps 1 \
- --output_dir ./ \
- --device gpu >${log_path}/ofa_pretrain) >>${log_path}/ofa_pretrain 2>&1
-print_info $? ofa_pretrain
-mv sst-2_ft_model_1.pdparams/ ${nlp_dir}/examples/model_compression/ofa/
-cd -
-#model slim
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-time (python -m paddle.distributed.launch run_glue_ofa.py \
- --model_type bert \
- --model_name_or_path ./sst-2_ft_model_1.pdparams/ \
- --task_name SST-2 --max_seq_length 128 \
- --batch_size 32 \
- --learning_rate 2e-5 \
- --num_train_epochs 1 \
- --max_steps 1 \
- --logging_steps 1 \
- --save_steps 1 \
- --output_dir ./ofa/SST-2 \
- --device gpu \
- --width_mult_list 1.0 0.8333333333333334 0.6666666666666666 0.5 >${log_path}/ofa_slim) >>${log_path}/ofa_slim 2>&1
-print_info $? ofa_slim
+ cd ${nlp_dir}/examples/model_compression/ofa/
+ cd ../../benchmark/glue/
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ # finetuing
+ time (python -u run_glue.py \
+ --model_type bert \
+ --model_name_or_path bert-base-uncased \
+ --task_name SST-2 \
+ --max_seq_length 128 \
+ --batch_size 32 \
+ --learning_rate 2e-5 \
+ --num_train_epochs 1 \
+ --max_steps 1 \
+ --logging_steps 1 \
+ --save_steps 1 \
+ --output_dir ./ \
+ --device gpu >${log_path}/ofa_pretrain) >>${log_path}/ofa_pretrain 2>&1
+ print_info $? ofa_pretrain
+ mv sst-2_ft_model_1.pdparams/ ${nlp_dir}/examples/model_compression/ofa/
+ cd -
+ #model slim
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ time (python -m paddle.distributed.launch run_glue_ofa.py \
+ --model_type bert \
+ --model_name_or_path ./sst-2_ft_model_1.pdparams/ \
+ --task_name SST-2 --max_seq_length 128 \
+ --batch_size 32 \
+ --learning_rate 2e-5 \
+ --num_train_epochs 1 \
+ --max_steps 1 \
+ --logging_steps 1 \
+ --save_steps 1 \
+ --output_dir ./ofa/SST-2 \
+ --device gpu \
+ --width_mult_list 1.0 0.8333333333333334 0.6666666666666666 0.5 >${log_path}/ofa_slim) >>${log_path}/ofa_slim 2>&1
+ print_info $? ofa_slim
}
# 12 albert
-albert (){
-cd ${nlp_dir}/examples/benchmark/glue/
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-time (python -m paddle.distributed.launch run_glue.py \
- --model_type albert \
- --model_name_or_path albert-base-v2 \
+albert() {
+ cd ${nlp_dir}/legacy/examples/benchmark/glue/
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ time (python -m paddle.distributed.launch run_glue.py \
+ --model_type albert \
+ --model_name_or_path albert-base-v2 \
--task_name SST-2 \
- --max_seq_length 128 \
- --batch_size 32 \
- --learning_rate 1e-5 \
- --max_steps 1 \
- --warmup_steps 1256 \
- --logging_steps 1 \
- --save_steps 1 \
- --output_dir ./albert/SST-2/ \
+ --max_seq_length 128 \
+ --batch_size 32 \
+ --learning_rate 1e-5 \
+ --max_steps 1 \
+ --warmup_steps 1256 \
+ --logging_steps 1 \
+ --save_steps 1 \
+ --output_dir ./albert/SST-2/ \
--device gpu >${log_path}/albert_sst-2_train) >>${log_path}/albert_sst-2_train 2>&1
-print_info $? albert_sst-2_train
+ print_info $? albert_sst-2_train
}
# 13 squad
-squad (){
-cd ${nlp_dir}/examples/machine_reading_comprehension/SQuAD/
-export CUDA_VISIBLE_DEVICES=${cudaid1}
-# finetune
-time (python -m paddle.distributed.launch run_squad.py \
- --model_type bert \
- --model_name_or_path bert-base-uncased \
- --max_seq_length 384 \
- --batch_size 12 \
- --learning_rate 3e-5 \
- --num_train_epochs 1 \
- --max_steps 1 \
- --logging_steps 1 \
- --save_steps 1 \
- --warmup_proportion 0.1 \
- --weight_decay 0.01 \
- --output_dir ./tmp/squad/ \
- --device gpu \
- --do_train \
- --do_predict >${log_path}/squad_train) >>${log_path}/squad_train 2>&1
-print_info $? squad_train
-# export model
-time (python -u ./export_model.py \
- --model_type bert \
- --model_path ./tmp/squad/model_1/ \
- --output_path ./infer_model/model >${log_path}/squad_export) >>${log_path}/squad_export 2>&1
-print_info $? squad_export
-# predict
-time (python -u deploy/python/predict.py \
- --model_type bert \
- --model_name_or_path ./infer_model/model \
- --batch_size 2 \
- --max_seq_length 384 >${log_path}/squad_predict) >>${log_path}/squad_predict 2>&1
-print_info $? squad_predict
+squad() {
+ cd ${nlp_dir}/legacy/examples/machine_reading_comprehension/SQuAD/
+ export CUDA_VISIBLE_DEVICES=${cudaid1}
+ # finetune
+ time (python -m paddle.distributed.launch run_squad.py \
+ --model_type bert \
+ --model_name_or_path bert-base-uncased \
+ --max_seq_length 384 \
+ --batch_size 12 \
+ --learning_rate 3e-5 \
+ --num_train_epochs 1 \
+ --max_steps 1 \
+ --logging_steps 1 \
+ --save_steps 1 \
+ --warmup_proportion 0.1 \
+ --weight_decay 0.01 \
+ --output_dir ./tmp/squad/ \
+ --device gpu \
+ --do_train \
+ --do_predict >${log_path}/squad_train) >>${log_path}/squad_train 2>&1
+ print_info $? squad_train
+ # export model
+ time (python -u ./export_model.py \
+ --model_type bert \
+ --model_path ./tmp/squad/model_1/ \
+ --output_path ./infer_model/model >${log_path}/squad_export) >>${log_path}/squad_export 2>&1
+ print_info $? squad_export
+ # predict
+ time (python -u deploy/python/predict.py \
+ --model_type bert \
+ --model_name_or_path ./infer_model/model \
+ --batch_size 2 \
+ --max_seq_length 384 >${log_path}/squad_predict) >>${log_path}/squad_predict 2>&1
+ print_info $? squad_predict
}
# 15 lexical_analysis
lexical_analysis(){
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-cd ${nlp_dir}/examples/lexical_analysis/
-#train
-time (python download.py --data_dir ./ )
-time (python -m paddle.distributed.launch train.py \
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ cd ${nlp_dir}/examples/lexical_analysis/
+ #train
+ time (python download.py --data_dir ./ )
+ time (python -m paddle.distributed.launch train.py \
--data_dir ./lexical_analysis_dataset_tiny \
--model_save_dir ./save_dir \
--epochs 1 \
@@ -435,643 +419,475 @@ time (python -m paddle.distributed.launch train.py \
--logging_steps 1\
--batch_size 32 \
--device gpu >${log_path}/lexical_analysis_train) >>${log_path}/lexical_analysis_train 2>&1
-print_info $? lexical_analysis_train
-#export
-time (python export_model.py \
- --data_dir=./lexical_analysis_dataset_tiny \
- --params_path=./save_dir/model_15.pdparams \
- --output_path=./infer_model/static_graph_params >${log_path}/lexical_analysis_export) >>${log_path}/lexical_analysis_export 2>&1
-print_info $? lexical_analysis_export
-# predict
-time (python predict.py --data_dir ./lexical_analysis_dataset_tiny \
+ print_info $? lexical_analysis_train
+ #export
+ time (python export_model.py \
+ --data_dir=./lexical_analysis_dataset_tiny \
+ --params_path=./save_dir/model_15.pdparams \
+ --output_path=./infer_model/static_graph_params >${log_path}/lexical_analysis_export) >>${log_path}/lexical_analysis_export 2>&1
+ print_info $? lexical_analysis_export
+ # predict
+ time (python predict.py --data_dir ./lexical_analysis_dataset_tiny \
--init_checkpoint ./save_dir/model_15.pdparams \
--batch_size 32 \
--device gpu >${log_path}/lexical_analysis_predict) >>${log_path}/lexical_analysis_predict 2>&1
-print_info $? lexical_analysis_predict
-# deploy
-time (python deploy/predict.py \
- --model_file=infer_model/static_graph_params.pdmodel \
- --params_file=infer_model/static_graph_params.pdiparams \
- --data_dir lexical_analysis_dataset_tiny >${log_path}/lexical_analysis_deploy) >>${log_path}/lexical_analysis_deploy 2>&1
-print_info $? lexical_analysis_deploy
-}
-# 16 seq2seq
-seq2seq() {
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-cd ${nlp_dir}/examples/machine_translation/seq2seq/
-# train (1041/steps) 5min
-time (python train.py \
- --num_layers 2 \
- --hidden_size 512 \
- --batch_size 128 \
- --max_epoch 1 \
- --log_freq 1 \
- --dropout 0.2 \
- --init_scale 0.1 \
- --max_grad_norm 5.0 \
- --device gpu \
- --model_path ./attention_models >${log_path}/seq2seq_train) >>${log_path}/seq2seq_train 2>&1
-print_info $? seq2seq_train
-# predict
-time (python predict.py \
- --num_layers 2 \
- --hidden_size 512 \
- --batch_size 128 \
- --dropout 0.2 \
- --init_scale 0.1 \
- --max_grad_norm 5.0 \
- --init_from_ckpt attention_models/0 \
- --infer_output_file infer_output.txt \
- --beam_size 10 \
- --device gpu >${log_path}/seq2seq_predict) >>${log_path}/seq2seq_predict 2>&1
-print_info $? seq2seq_predict
-# export
-time (python export_model.py \
- --num_layers 2 \
- --hidden_size 512 \
- --batch_size 128 \
- --dropout 0.2 \
- --init_scale 0.1 \
- --max_grad_norm 5.0 \
- --init_from_ckpt attention_models/0.pdparams \
- --beam_size 10 \
- --export_path ./infer_model/model >${log_path}/seq2seq_export) >>${log_path}/seq2seq_export 2>&1
-print_info $? seq2seq_export
-# depoly
-time (cd deploy/python
-python infer.py \
- --export_path ../../infer_model/model \
- --device gpu \
- --batch_size 128 \
- --infer_output_file infer_output.txt >${log_path}/seq2seq_depoly) >>${log_path}/seq2seq_deploy 2>&1
-print_info $? seq2seq_depoly
+ print_info $? lexical_analysis_predict
+ # deploy
+ time (python deploy/predict.py \
+ --model_file=infer_model/static_graph_params.pdmodel \
+ --params_file=infer_model/static_graph_params.pdiparams \
+ --data_dir lexical_analysis_dataset_tiny >${log_path}/lexical_analysis_deploy) >>${log_path}/lexical_analysis_deploy 2>&1
+ print_info $? lexical_analysis_deploy
}
# 18 word_embedding 5min
word_embedding(){
-export CUDA_VISIBLE_DEVICES=${cudaid1}
-cd ${nlp_dir}/examples/word_embedding/
-# 使用paddlenlp.embeddings.TokenEmbedding
-time (python train.py --device='gpu' \
- --lr=5e-4 \
- --batch_size=32 \
- --epochs=1 \
- --use_token_embedding=True \
- --vdl_dir='./vdl_paddlenlp_dir' >${log_path}/word_embedding_paddlenlp_train) >>${log_path}/word_embedding_paddlenlp_train 2>&1
-print_info $? word_embedding_paddlenlp_train
-# 使用paddle.nn.Embedding
-time (python train.py --device='gpu' \
- --lr=1e-4 \
- --batch_size=32 \
- --epochs=1 \
- --use_token_embedding=False \
- --vdl_dir='./vdl_paddle_dir' >${log_path}/word_embedding_paddle_train) >>${log_path}/word_embedding_paddle_train 2>&1
-print_info $? word_embedding_paddle_train
-}
-# 19 ernie-ctm
-ernie-ctm(){
-export CUDA_VISIBLE_DEVICES=${cudaid1}
-cd ${nlp_dir}/examples/text_to_knowledge/ernie-ctm/
-wget https://paddlenlp.bj.bcebos.com/paddlenlp/datasets/wordtag_dataset_v2.tar.gz && tar -zxvf wordtag_dataset_v2.tar.gz
-time (python -m paddle.distributed.launch train.py \
- --max_seq_len 128 \
- --batch_size 8 \
- --learning_rate 5e-5 \
- --num_train_epochs 1 \
- --logging_steps 1 \
- --save_steps 100 \
- --output_dir ./output/ \
- --device "gpu" >${log_path}/ernie-ctm_train) >>${log_path}/ernie-ctm_train 2>&1
-print_info $? ernie-ctm_train
-export CUDA_VISIBLE_DEVICES=${cudaid1}
-time (python -m paddle.distributed.launch predict.py \
- --batch_size 32 \
- --params_path ./output/model_125/model_state.pdparams \
- --device "gpu" >${log_path}/ernie-ctm_eval) >>${log_path}/ernie-ctm_eval 2>&1
-print_info $? ernie-ctm_eval
-}
-# 20 distilbert
-distilbert (){
-cd ${nlp_dir}/examples/model_compression/distill_lstm/
-wget -q https://paddle-qa.bj.bcebos.com/SST-2_GLUE.tar
-tar -xzvf SST-2_GLUE.tar
-time (
- python small.py \
- --task_name sst-2 \
- --vocab_size 30522 \
- --max_epoch 1 \
- --batch_size 64 \
- --lr 1.0 \
- --dropout_prob 0.4 \
- --output_dir small_models/SST-2 \
- --save_steps 10000 \
- --embedding_name w2v.google_news.target.word-word.dim300.en >${log_path}/distilbert_small_train) >>${log_path}/distilbert_small_train 2>&1
-print_info $? distilbert_small_train
-time (
- python bert_distill.py \
- --task_name sst-2 \
- --vocab_size 30522 \
- --max_epoch 1 \
- --lr 1.0 \
- --task_name sst-2 \
- --dropout_prob 0.2 \
- --batch_size 128 \
- --model_name bert-base-uncased \
- --output_dir distilled_models/SST-2 \
- --teacher_dir ./SST-2/sst-2_ft_model_1.pdparams/ \
- --save_steps 1000 \
- --n_iter 1 \
- --embedding_name w2v.google_news.target.word-word.dim300.en >${log_path}/distilbert_teacher_train) >>${log_path}/distilbert_teacher_train 2>&1
-print_info $? distilbert_teacher_train
-}
-fast_transformer(){
-# FT
-cd ${nlp_dir}/
-export PYTHONPATH=$PWD/PaddleNLP/:$PYTHONPATH
-wget -q https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-TagBuild-Infer-Linux-Gpu-Cuda120-Cudnn89-Trt86-Mkl-Avx-Gcc122/latest/paddle_inference.tgz
-tar -zxf paddle_inference.tgz
-cd ${nlp_dir}/paddlenlp/ops
-#python op
-mkdir build_tr_so
-cd build_tr_so/
-cmake .. -DCMAKE_BUILD_TYPE=Release \
--DCMAKE_C_COMPILER=${C_COMPILER_PATH} \
--DCMAKE_CXX_COMPILER=${CXX_COMPILER_PATH} \
--DPY_CMD=python \
--DPADDLE_LIB=${nlp_dir}/paddle_inference \
--DDEMO=${nlp_dir}/paddlenlp/ops/fast_transformer/src/demo/transformer_e2e.cc \
--DON_INFER=ON -DWITH_MKL=ON -DWITH_ONNXRUNTIME=ON
-make -j >${log_path}/transformer_python_FT >>${log_path}/transformer_python_FT 2>&1
-print_info $? transformer_python_FT
-cd ../
-#C++ op
-mkdir build_tr_cc
-cd build_tr_cc/
-cmake .. -DCMAKE_BUILD_TYPE=Release \
--DCMAKE_C_COMPILER=${C_COMPILER_PATH} \
--DCMAKE_CXX_COMPILER=${CXX_COMPILER_PATH} \
--DPADDLE_LIB=${nlp_dir}/paddle_inference -DDEMO=${nlp_dir}/paddlenlp/ops/fast_transformer/src/demo/transformer_e2e.cc \
--DON_INFER=ON -DWITH_MKL=ON -DWITH_ONNXRUNTIME=ON
-make -j >${log_path}/transformer_C_FT >>${log_path}/transformer_C_FT 2>&1
-print_info $? transformer_C_FT
-#deploy python
-cd ${nlp_dir}/examples/machine_translation/transformer/fast_transformer/
-sed -i "s#./trained_models/step_final/#./base_trained_models/step_final/#g" ../configs/transformer.base.yaml
-wget -q https://paddlenlp.bj.bcebos.com/models/transformers/transformer/transformer-base-wmt_ende_bpe.tar.gz
-tar -zxf transformer-base-wmt_ende_bpe.tar.gz
-export FLAGS_fraction_of_gpu_memory_to_use=0.1
-cp -rf ${nlp_dir}/paddlenlp/ops/build_tr_so/third-party/build/fastertransformer/bin/decoding_gemm ./
-./decoding_gemm 8 4 8 64 38512 32 512 0
-#beam_search
-python encoder_decoding_predict.py \
- --config ../configs/transformer.base.yaml \
- --decoding_lib ${nlp_dir}/paddlenlp/ops/build_tr_so/lib/libdecoding_op.so \
- --decoding_strategy beam_search \
- --beam_size 5 >${log_path}/transformer_deploy_P_FT >>${log_path}/transformer_deploy_P_FT 2>&1
-print_info $? transformer_deploy_P_FT
-#topk
-python encoder_decoding_predict.py \
- --config ../configs/transformer.base.yaml \
- --decoding_lib ${nlp_dir}/paddlenlp/ops/build_tr_so/lib/libdecoding_op.so \
- --decoding_strategy topk_sampling \
- --topk 3 >topk.log
-#topp
-python encoder_decoding_predict.py \
- --config ../configs/transformer.base.yaml \
- --decoding_lib ${nlp_dir}/paddlenlp/ops/build_tr_so/lib/libdecoding_op.so \
- --decoding_strategy topp_sampling \
- --topk 0 \
- --topp 0.1 >topp.log
-#deploy c++
-python export_model.py \
- --config ../configs/transformer.base.yaml \
- --decoding_lib ${nlp_dir}/paddlenlp/ops/build_tr_so/lib/libdecoding_op.so \
- --decoding_strategy beam_search --beam_size 5
-./decoding_gemm 8 5 8 64 38512 256 512 0
-${nlp_dir}/paddlenlp/ops/build_tr_cc/bin/./transformer_e2e -batch_size 8 -gpu_id 0 -model_dir ./infer_model/ -vocab_file ${PPNLP_HOME}/datasets/WMT14ende/WMT14.en-de/wmt14_ende_data_bpe/vocab_all.bpe.33708 \
--data_file ${PPNLP_HOME}/datasets/WMT14ende/WMT14.en-de/wmt14_ende_data_bpe/newstest2014.tok.bpe.33708.en >${log_path}/transformer_deploy_C_FT >>${log_path}/transformer_deploy_C_FT 2>&1
-print_info $? transformer_deploy_C_FT
+ export CUDA_VISIBLE_DEVICES=${cudaid1}
+ cd ${nlp_dir}/examples/word_embedding/
+ # 使用paddlenlp.embeddings.TokenEmbedding
+ time (python train.py --device='gpu' \
+ --lr=5e-4 \
+ --batch_size=32 \
+ --epochs=1 \
+ --use_token_embedding=True \
+ --vdl_dir='./vdl_paddlenlp_dir' >${log_path}/word_embedding_paddlenlp_train) >>${log_path}/word_embedding_paddlenlp_train 2>&1
+ print_info $? word_embedding_paddlenlp_train
+ # 使用paddle.nn.Embedding
+ time (python train.py --device='gpu' \
+ --lr=1e-4 \
+ --batch_size=32 \
+ --epochs=1 \
+ --use_token_embedding=False \
+ --vdl_dir='./vdl_paddle_dir' >${log_path}/word_embedding_paddle_train) >>${log_path}/word_embedding_paddle_train 2>&1
+ print_info $? word_embedding_paddle_train
+}
+fast_transformer() {
+ # FT
+ cd ${nlp_dir}/
+ export PYTHONPATH=$PWD/PaddleNLP/:$PYTHONPATH
+ wget -q https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-TagBuild-Infer-Linux-Gpu-Cuda120-Cudnn89-Trt86-Mkl-Avx-Gcc122/latest/paddle_inference.tgz
+ tar -zxf paddle_inference.tgz
+ cd ${nlp_dir}/paddlenlp/ops
+ #python op
+ mkdir build_tr_so
+ cd build_tr_so/
+ cmake .. -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_C_COMPILER=${C_COMPILER_PATH} \
+ -DCMAKE_CXX_COMPILER=${CXX_COMPILER_PATH} \
+ -DPY_CMD=python \
+ -DPADDLE_LIB=${nlp_dir}/paddle_inference \
+ -DDEMO=${nlp_dir}/paddlenlp/ops/fast_transformer/src/demo/transformer_e2e.cc \
+ -DON_INFER=ON -DWITH_MKL=ON -DWITH_ONNXRUNTIME=ON
+ make -j >${log_path}/transformer_python_FT >>${log_path}/transformer_python_FT 2>&1
+ print_info $? transformer_python_FT
+ cd ../
+ #C++ op
+ mkdir build_tr_cc
+ cd build_tr_cc/
+ cmake .. -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_C_COMPILER=${C_COMPILER_PATH} \
+ -DCMAKE_CXX_COMPILER=${CXX_COMPILER_PATH} \
+ -DPADDLE_LIB=${nlp_dir}/paddle_inference -DDEMO=${nlp_dir}/paddlenlp/ops/fast_transformer/src/demo/transformer_e2e.cc \
+ -DON_INFER=ON -DWITH_MKL=ON -DWITH_ONNXRUNTIME=ON
+ make -j >${log_path}/transformer_C_FT >>${log_path}/transformer_C_FT 2>&1
+ print_info $? transformer_C_FT
+ #deploy python
+ cd ${nlp_dir}/examples/machine_translation/transformer/fast_transformer/
+ sed -i "s#./trained_models/step_final/#./base_trained_models/step_final/#g" ../configs/transformer.base.yaml
+ wget -q https://paddlenlp.bj.bcebos.com/models/transformers/transformer/transformer-base-wmt_ende_bpe.tar.gz
+ tar -zxf transformer-base-wmt_ende_bpe.tar.gz
+ export FLAGS_fraction_of_gpu_memory_to_use=0.1
+ cp -rf ${nlp_dir}/paddlenlp/ops/build_tr_so/third-party/build/fastertransformer/bin/decoding_gemm ./
+ ./decoding_gemm 8 4 8 64 38512 32 512 0
+ #beam_search
+ python encoder_decoding_predict.py \
+ --config ../configs/transformer.base.yaml \
+ --decoding_lib ${nlp_dir}/paddlenlp/ops/build_tr_so/lib/libdecoding_op.so \
+ --decoding_strategy beam_search \
+ --beam_size 5 >${log_path}/transformer_deploy_P_FT >>${log_path}/transformer_deploy_P_FT 2>&1
+ print_info $? transformer_deploy_P_FT
+ #topk
+ python encoder_decoding_predict.py \
+ --config ../configs/transformer.base.yaml \
+ --decoding_lib ${nlp_dir}/paddlenlp/ops/build_tr_so/lib/libdecoding_op.so \
+ --decoding_strategy topk_sampling \
+ --topk 3 >topk.log
+ #topp
+ python encoder_decoding_predict.py \
+ --config ../configs/transformer.base.yaml \
+ --decoding_lib ${nlp_dir}/paddlenlp/ops/build_tr_so/lib/libdecoding_op.so \
+ --decoding_strategy topp_sampling \
+ --topk 0 \
+ --topp 0.1 >topp.log
+ #deploy c++
+ python export_model.py \
+ --config ../configs/transformer.base.yaml \
+ --decoding_lib ${nlp_dir}/paddlenlp/ops/build_tr_so/lib/libdecoding_op.so \
+ --decoding_strategy beam_search --beam_size 5
+ ./decoding_gemm 8 5 8 64 38512 256 512 0
+ ${nlp_dir}/paddlenlp/ops/build_tr_cc/bin/./transformer_e2e -batch_size 8 -gpu_id 0 -model_dir ./infer_model/ -vocab_file ${PPNLP_HOME}/datasets/WMT14ende/WMT14.en-de/wmt14_ende_data_bpe/vocab_all.bpe.33708 \
+ -data_file ${PPNLP_HOME}/datasets/WMT14ende/WMT14.en-de/wmt14_ende_data_bpe/newstest2014.tok.bpe.33708.en >${log_path}/transformer_deploy_C_FT >>${log_path}/transformer_deploy_C_FT 2>&1
+ print_info $? transformer_deploy_C_FT
}
# 22 transformer
-transformer (){
-cd ${nlp_dir}/examples/machine_translation/transformer/
-wget -q https://paddle-qa.bj.bcebos.com/paddlenlp/WMT14.en-de.partial.tar.gz
-tar -xzvf WMT14.en-de.partial.tar.gz
-time (
-sed -i "s/save_step: 10000/save_step: 1/g" configs/transformer.base.yaml
-sed -i "s/print_step: 100/print_step: 1/g" configs/transformer.base.yaml
-sed -i "s/epoch: 30/epoch: 1/g" configs/transformer.base.yaml
-sed -i "s/max_iter: None/max_iter: 2/g" configs/transformer.base.yaml
-sed -i "s/batch_size: 4096/batch_size: 1000/g" configs/transformer.base.yaml
+transformer() {
+ cd ${nlp_dir}/legacy/examples/machine_translation/transformer/
+ wget -q https://paddle-qa.bj.bcebos.com/paddlenlp/WMT14.en-de.partial.tar.gz
+ tar -xzvf WMT14.en-de.partial.tar.gz
+ time (
+ sed -i "s/save_step: 10000/save_step: 1/g" configs/transformer.base.yaml
+ sed -i "s/print_step: 100/print_step: 1/g" configs/transformer.base.yaml
+ sed -i "s/epoch: 30/epoch: 1/g" configs/transformer.base.yaml
+ sed -i "s/max_iter: None/max_iter: 2/g" configs/transformer.base.yaml
+ sed -i "s/batch_size: 4096/batch_size: 1000/g" configs/transformer.base.yaml
-python train.py --config ./configs/transformer.base.yaml \
- --train_file ${PWD}/WMT14.en-de.partial/train.tok.clean.bpe.en ${PWD}/WMT14.en-de.partial/train.tok.clean.bpe.de \
- --dev_file ${PWD}/WMT14.en-de.partial/dev.tok.bpe.en ${PWD}/WMT14.en-de.partial/dev.tok.bpe.de \
- --vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
- --unk_token "" --bos_token "" --eos_token "" >${log_path}/transformer_train) >>${log_path}/transformer_train 2>&1
-print_info $? transformer_train
-#predict
-time (
-sed -i 's#init_from_params: "./trained_models/step/"#init_from_params: "./trained_models/step_final/"#g' configs/transformer.base.yaml
-python predict.py --config ./configs/transformer.base.yaml \
- --test_file ${PWD}/WMT14.en-de.partial/test.tok.bpe.en ${PWD}/WMT14.en-de.partial/test.tok.bpe.de \
- --without_ft \
- --vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
- --unk_token "" --bos_token "" --eos_token "" >${log_path}/transformer_predict) >>${log_path}/transformer_predict 2>&1
-print_info $? transformer_predict
-#export
-time (
-python export_model.py --config ./configs/transformer.base.yaml \
- --vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
- --bos_token "" --eos_token "" >${log_path}/transformer_export) >>${log_path}/transformer_export 2>&1
-print_info $? transformer_export
-#infer
-time (
-python ./deploy/python/inference.py --config ./configs/transformer.base.yaml \
- --profile \
- --test_file ${PWD}/WMT14.en-de.partial/test.tok.bpe.en ${PWD}/WMT14.en-de.partial/test.tok.bpe.de \
- --vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
- --unk_token "" --bos_token "" --eos_token "" >${log_path}/transformer_infer) >>${log_path}/transformer_infer 2>&1
-print_info $? transformer_infer
+ python train.py --config ./configs/transformer.base.yaml \
+ --train_file ${PWD}/WMT14.en-de.partial/train.tok.clean.bpe.en ${PWD}/WMT14.en-de.partial/train.tok.clean.bpe.de \
+ --dev_file ${PWD}/WMT14.en-de.partial/dev.tok.bpe.en ${PWD}/WMT14.en-de.partial/dev.tok.bpe.de \
+ --vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
+ --unk_token "" --bos_token "" --eos_token "" >${log_path}/transformer_train
+ ) >>${log_path}/transformer_train 2>&1
+ print_info $? transformer_train
+ #predict
+ time (
+ sed -i 's#init_from_params: "./trained_models/step/"#init_from_params: "./trained_models/step_final/"#g' configs/transformer.base.yaml
+ python predict.py --config ./configs/transformer.base.yaml \
+ --test_file ${PWD}/WMT14.en-de.partial/test.tok.bpe.en ${PWD}/WMT14.en-de.partial/test.tok.bpe.de \
+ --without_ft \
+ --vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
+ --unk_token "" --bos_token "" --eos_token "" >${log_path}/transformer_predict
+ ) >>${log_path}/transformer_predict 2>&1
+ print_info $? transformer_predict
+ #export
+ time (
+ python export_model.py --config ./configs/transformer.base.yaml \
+ --vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
+ --bos_token "" --eos_token "" >${log_path}/transformer_export
+ ) >>${log_path}/transformer_export 2>&1
+ print_info $? transformer_export
+ #infer
+ time (
+ python ./deploy/python/inference.py --config ./configs/transformer.base.yaml \
+ --profile \
+ --test_file ${PWD}/WMT14.en-de.partial/test.tok.bpe.en ${PWD}/WMT14.en-de.partial/test.tok.bpe.de \
+ --vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
+ --unk_token "" --bos_token "" --eos_token "" >${log_path}/transformer_infer
+ ) >>${log_path}/transformer_infer 2>&1
+ print_info $? transformer_infer
-# fast_transformer
-}
-# 23 pet
-pet (){
-path="examples/few_shot/pet"
-python scripts/regression/ci_normal_case.py ${path}
-}
-efl(){
-path="examples/few_shot/efl"
-python scripts/regression/ci_normal_case.py ${path}
-}
-p-tuning(){
-path="examples/few_shot/p-tuning"
-python scripts/regression/ci_normal_case.py ${path}
+ # fast_transformer
}
#25 ernie-doc
ernie-doc(){
-cd ${nlp_dir}/model_zoo/ernie-doc/
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-time (python -m paddle.distributed.launch --log_dir hyp run_classifier.py --epochs 15 --layerwise_decay 0.7 --learning_rate 5e-5 --batch_size 4 --save_steps 100 --max_steps 100 --dataset hyp --output_dir hyp >${log_path}/ernie-doc_hyp) >>${log_path}/ernie-doc_hyp 2>&1
-print_info $? ernie-doc_hyp
-time (python -m paddle.distributed.launch --log_dir cmrc2018 run_mrc.py --batch_size 4 --layerwise_decay 0.8 --dropout 0.2 --learning_rate 4.375e-5 --epochs 1 --save_steps 100 --max_steps 100 --dataset cmrc2018 --output_dir cmrc2018 >${log_path}/ernie-doc_cmrc2018) >>${log_path}/ernie-doc_cmrc2018 2>&1
-print_info $? ernie-doc_cmrc2018
-time (python -m paddle.distributed.launch --log_dir c3 run_mcq.py --learning_rate 6.5e-5 --epochs 1 --save_steps 100 --max_steps 100 --output_dir c3 >${log_path}/ernie-doc_c3) >>${log_path}/ernie-doc_c3 2>&1
-print_info $? ernie-doc_c3
-time (python -m paddle.distributed.launch --log_dir cail/ run_semantic_matching.py --epochs 1 --layerwise_decay 0.8 --learning_rate 1.25e-5 --batch_size 4 --save_steps 100 --max_steps 100 --output_dir cail >${log_path}/ernie-doc_cail) >>${log_path}/ernie-doc_cail 2>&1
-print_info $? ernie-doc_cail
-time (python -m paddle.distributed.launch --log_dir msra run_sequence_labeling.py --learning_rate 3e-5 --epochs 1 --save_steps 100 --max_steps 100 --output_dir msra >${log_path}/ernie-doc_msar) >>${log_path}/ernie-doc_msar 2>&1
-print_info $? ernie-doc_msar
-time (python run_mrc.py --model_name_or_path ernie-doc-base-zh --dataset dureader_robust --batch_size 8 --learning_rate 2.75e-4 --epochs 1 --save_steps 10 --max_steps 2 --logging_steps 10 --device gpu >${log_path}/ernie-doc_dureader_robust) >>${log_path}/ernie-doc_dureader_robust 2>&1
-print_info $? ernie-doc_dureader_robust
+ cd ${nlp_dir}/model_zoo/ernie-doc/
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ time (python -m paddle.distributed.launch --log_dir hyp run_classifier.py --epochs 15 --layerwise_decay 0.7 --learning_rate 5e-5 --batch_size 4 --save_steps 100 --max_steps 100 --dataset hyp --output_dir hyp >${log_path}/ernie-doc_hyp) >>${log_path}/ernie-doc_hyp 2>&1
+ print_info $? ernie-doc_hyp
+ time (python -m paddle.distributed.launch --log_dir cmrc2018 run_mrc.py --batch_size 4 --layerwise_decay 0.8 --dropout 0.2 --learning_rate 4.375e-5 --epochs 1 --save_steps 100 --max_steps 100 --dataset cmrc2018 --output_dir cmrc2018 >${log_path}/ernie-doc_cmrc2018) >>${log_path}/ernie-doc_cmrc2018 2>&1
+ print_info $? ernie-doc_cmrc2018
+ time (python -m paddle.distributed.launch --log_dir c3 run_mcq.py --learning_rate 6.5e-5 --epochs 1 --save_steps 100 --max_steps 100 --output_dir c3 >${log_path}/ernie-doc_c3) >>${log_path}/ernie-doc_c3 2>&1
+ print_info $? ernie-doc_c3
+ time (python -m paddle.distributed.launch --log_dir cail/ run_semantic_matching.py --epochs 1 --layerwise_decay 0.8 --learning_rate 1.25e-5 --batch_size 4 --save_steps 100 --max_steps 100 --output_dir cail >${log_path}/ernie-doc_cail) >>${log_path}/ernie-doc_cail 2>&1
+ print_info $? ernie-doc_cail
+ time (python -m paddle.distributed.launch --log_dir msra run_sequence_labeling.py --learning_rate 3e-5 --epochs 1 --save_steps 100 --max_steps 100 --output_dir msra >${log_path}/ernie-doc_msar) >>${log_path}/ernie-doc_msar 2>&1
+ print_info $? ernie-doc_msar
+ time (python run_mrc.py --model_name_or_path ernie-doc-base-zh --dataset dureader_robust --batch_size 8 --learning_rate 2.75e-4 --epochs 1 --save_steps 10 --max_steps 2 --logging_steps 10 --device gpu >${log_path}/ernie-doc_dureader_robust) >>${log_path}/ernie-doc_dureader_robust 2>&1
+ print_info $? ernie-doc_dureader_robust
}
#26 transformer-xl
transformer-xl (){
-cd ${nlp_dir}/examples/language_model/transformer-xl/
-mkdir gen_data && cd gen_data
-wget https://paddle-qa.bj.bcebos.com/paddlenlp/enwik8.tar.gz && tar -zxvf enwik8.tar.gz
-cd ../
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-time (sed -i 's/print_step: 100/print_step: 1/g' configs/enwik8.yaml
-sed -i 's/save_step: 10000/save_step: 3/g' configs/enwik8.yaml
-sed -i 's/batch_size: 16/batch_size: 8/g' configs/enwik8.yaml
-sed -i 's/max_step: 400000/max_step: 3/g' configs/enwik8.yaml
-python -m paddle.distributed.launch train.py --config ./configs/enwik8.yaml >${log_path}/transformer-xl_train_enwik8) >>${log_path}/transformer-xl_train_enwik8 2>&1
-print_info $? transformer-xl_train_enwik8
-time (sed -i 's/batch_size: 8/batch_size: 1/g' configs/enwik8.yaml
-sed -i 's#init_from_params: "./trained_models/step_final/"#init_from_params: "./trained_models/step_3/"#g' configs/enwik8.yaml
-python eval.py --config ./configs/enwik8.yaml >${log_path}/transformer-xl_eval_enwik8) >>${log_path}/transformer-xl_eval_enwik8 2>&1
-print_info $? transformer-xl_eval_enwik8
+ cd ${nlp_dir}/examples/language_model/transformer-xl/
+ mkdir gen_data && cd gen_data
+ wget https://paddle-qa.bj.bcebos.com/paddlenlp/enwik8.tar.gz && tar -zxvf enwik8.tar.gz
+ cd ../
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ time (sed -i 's/print_step: 100/print_step: 1/g' configs/enwik8.yaml
+ sed -i 's/save_step: 10000/save_step: 3/g' configs/enwik8.yaml
+ sed -i 's/batch_size: 16/batch_size: 8/g' configs/enwik8.yaml
+ sed -i 's/max_step: 400000/max_step: 3/g' configs/enwik8.yaml
+ python -m paddle.distributed.launch train.py --config ./configs/enwik8.yaml >${log_path}/transformer-xl_train_enwik8) >>${log_path}/transformer-xl_train_enwik8 2>&1
+ print_info $? transformer-xl_train_enwik8
+ time (sed -i 's/batch_size: 8/batch_size: 1/g' configs/enwik8.yaml
+ sed -i 's#init_from_params: "./trained_models/step_final/"#init_from_params: "./trained_models/step_3/"#g' configs/enwik8.yaml
+ python eval.py --config ./configs/enwik8.yaml >${log_path}/transformer-xl_eval_enwik8) >>${log_path}/transformer-xl_eval_enwik8 2>&1
+ print_info $? transformer-xl_eval_enwik8
}
#28 question_matching
question_matching() {
-cd ${nlp_dir}/examples/text_matching/question_matching/
-wget -q https://paddle-qa.bj.bcebos.com/paddlenlp/data_v4.tar.gz
-tar -xvzf data_v4.tar.gz
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-#train
-time (
-python -u -m paddle.distributed.launch train.py \
- --train_set ./data_v4/train/ALL/train \
- --dev_set ./data_v4/train/ALL/dev \
- --device gpu \
- --eval_step 10 \
- --max_steps 10 \
- --save_dir ./checkpoints \
- --train_batch_size 32 \
- --learning_rate 2E-5 \
- --epochs 1 \
- --rdrop_coef 0.0 >${log_path}/question_matching_train) >>${log_path}/question_matching_train 2>&1
-print_info $? question_matching_train
-#predict
-time (
-export CUDA_VISIBLE_DEVICES=${cudaid1}
-python -u \
- predict.py \
- --device gpu \
- --params_path "./checkpoints/model_10/model_state.pdparams" \
- --batch_size 128 \
- --input_file ./data_v4/test/public_test_A \
- --result_file 0.0_predict_public_result_test_A_re >${log_path}/question_matching_predict) >>${log_path}/question_matching_predict 2>&1
-print_info $? question_matching_predict
+ cd ${nlp_dir}/examples/text_matching/question_matching/
+ wget -q https://paddle-qa.bj.bcebos.com/paddlenlp/data_v4.tar.gz
+ tar -xvzf data_v4.tar.gz
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ #train
+ time (
+ python -u -m paddle.distributed.launch train.py \
+ --train_set ./data_v4/train/ALL/train \
+ --dev_set ./data_v4/train/ALL/dev \
+ --device gpu \
+ --eval_step 10 \
+ --max_steps 10 \
+ --save_dir ./checkpoints \
+ --train_batch_size 32 \
+ --learning_rate 2E-5 \
+ --epochs 1 \
+ --rdrop_coef 0.0 >${log_path}/question_matching_train) >>${log_path}/question_matching_train 2>&1
+ print_info $? question_matching_train
+ #predict
+ time (
+ export CUDA_VISIBLE_DEVICES=${cudaid1}
+ python -u \
+ predict.py \
+ --device gpu \
+ --params_path "./checkpoints/model_10/model_state.pdparams" \
+ --batch_size 128 \
+ --input_file ./data_v4/test/public_test_A \
+ --result_file 0.0_predict_public_result_test_A_re >${log_path}/question_matching_predict) >>${log_path}/question_matching_predict 2>&1
+ print_info $? question_matching_predict
}
# 29 ernie-csc
ernie-csc() {
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-cd ${nlp_dir}/examples/text_correction/ernie-csc
-#dowdnload data
-python download.py --data_dir ./extra_train_ds/ --url https://github.com/wdimmy/Automatic-Corpus-Generation/raw/master/corpus/train.sgml
-#trans xml txt
-python change_sgml_to_txt.py -i extra_train_ds/train.sgml -o extra_train_ds/train.txt
-#2卡训练
-python -m paddle.distributed.launch train.py --batch_size 32 --logging_steps 100 --epochs 1 --learning_rate 5e-5 --model_name_or_path ernie-1.0-base-zh --output_dir ./checkpoints/ --extra_train_ds_dir ./extra_train_ds/ >${log_path}/ernie-csc_train >>${log_path}/ernie-csc_train 2>&1
-print_info $? ernie-csc_train
-#predict
-sh run_sighan_predict.sh >${log_path}/ernie-csc_predict >>${log_path}/ernie-csc_predict 2>&1
-print_info $? ernie-csc_predict
-#export model
-python export_model.py --params_path ./checkpoints/best_model.pdparams --output_path ./infer_model/static_graph_params >${log_path}/ernie-csc_export >>${log_path}/ernie-csc_export 2>&1
-print_info $? ernie-csc_export
-#python deploy
-python predict.py --model_file infer_model/static_graph_params.pdmodel --params_file infer_model/static_graph_params.pdiparams >${log_path}/ernie-csc_deploy >>${log_path}/ernie-csc_deploy 2>&1
-print_info $? ernie-csc_deploy
-}
-#30 nptag
-nptag() {
-cd ${nlp_dir}/examples/text_to_knowledge/nptag/
-wget -q https://paddlenlp.bj.bcebos.com/paddlenlp/datasets/nptag_dataset.tar.gz && tar -zxvf nptag_dataset.tar.gz
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-python -m paddle.distributed.launch train.py \
- --batch_size 64 \
- --learning_rate 1e-6 \
- --num_train_epochs 1 \
- --logging_steps 10 \
- --save_steps 100 \
- --output_dir ./output \
- --device "gpu" >${log_path}/nptag_train >>${log_path}/nptag_train 2>&1
-print_info $? nptag_train
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-python -m paddle.distributed.launch predict.py \
- --device=gpu \
- --params_path ./output/model_100/model_state.pdparams >${log_path}/nptag_predict >>${log_path}/nptag_predict 2>&1
-print_info $? nptag_predict
-python export_model.py --params_path=./output/model_100/model_state.pdparams --output_path=./export >${log_path}/nptag_export >>${log_path}/nptag_export 2>&1
-print_info $? nptag_export
-python deploy/python/predict.py --model_dir=./export >${log_path}/nptag_depoly >>${log_path}/nptag_deploy 2>&1
-print_info $? nptag_depoly
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ cd ${nlp_dir}/examples/text_correction/ernie-csc
+ #dowdnload data
+ python download.py --data_dir ./extra_train_ds/ --url https://github.com/wdimmy/Automatic-Corpus-Generation/raw/master/corpus/train.sgml
+ #trans xml txt
+ python change_sgml_to_txt.py -i extra_train_ds/train.sgml -o extra_train_ds/train.txt
+ #2卡训练
+ python -m paddle.distributed.launch train.py --batch_size 32 --logging_steps 100 --epochs 1 --learning_rate 5e-5 --model_name_or_path ernie-1.0-base-zh --output_dir ./checkpoints/ --extra_train_ds_dir ./extra_train_ds/ >${log_path}/ernie-csc_train >>${log_path}/ernie-csc_train 2>&1
+ print_info $? ernie-csc_train
+ #predict
+ sh run_sighan_predict.sh >${log_path}/ernie-csc_predict >>${log_path}/ernie-csc_predict 2>&1
+ print_info $? ernie-csc_predict
+ #export model
+ python export_model.py --params_path ./checkpoints/best_model.pdparams --output_path ./infer_model/static_graph_params >${log_path}/ernie-csc_export >>${log_path}/ernie-csc_export 2>&1
+ print_info $? ernie-csc_export
+ #python deploy
+ python predict.py --model_file infer_model/static_graph_params.pdmodel --params_file infer_model/static_graph_params.pdiparams >${log_path}/ernie-csc_deploy >>${log_path}/ernie-csc_deploy 2>&1
+ print_info $? ernie-csc_deploy
}
#31 ernie-m
ernie-m() {
-export CUDA_VISIBLE_DEVICES=${cudaid2}
-cd ${nlp_dir}/model_zoo/ernie-m
-# TODO(ouyanghongyu): remove the following scripts later.
-if [ ! -f 'test.py' ];then
- echo '模型测试文件不存在!'
- # finetuned for cross-lingual-transfer
- python -m paddle.distributed.launch --log_dir output_clt run_classifier.py \
- --do_train \
- --do_eval \
- --do_export \
+ export CUDA_VISIBLE_DEVICES=${cudaid2}
+ cd ${nlp_dir}/model_zoo/ernie-m
+ # TODO(ouyanghongyu): remove the following scripts later.
+ if [ ! -f 'test.py' ];then
+ echo '模型测试文件不存在!'
+ # finetuned for cross-lingual-transfer
+ python -m paddle.distributed.launch --log_dir output_clt run_classifier.py \
+ --do_train \
+ --do_eval \
+ --do_export \
+ --device gpu \
+ --task_type cross-lingual-transfer \
+ --model_name_or_path __internal_testing__/ernie-m \
+ --use_test_data True \
+ --test_data_path ../../tests/fixtures/tests_samples/xnli/xnli.jsonl \
+ --output_dir output_clt \
+ --export_model_dir output_clt \
+ --per_device_train_batch_size 8 \
+ --save_steps 1 \
+ --eval_steps 1 \
+ --max_steps 2 \
+ --overwrite_output_dir \
+ --remove_unused_columns False >${log_path}/ernie-m_clt >>${log_path}/ernie-m_clt 2>&1
+ print_info $? ernie-m_clt
+ # finetuned for translate-train-all
+ python -m paddle.distributed.launch --log_dir output_tta run_classifier.py \
+ --do_train \
+ --do_eval \
+ --do_export \
+ --device gpu \
+ --task_type translate-train-all \
+ --model_name_or_path __internal_testing__/ernie-m \
+ --use_test_data True \
+ --test_data_path ../../tests/fixtures/tests_samples/xnli/xnli.jsonl \
+ --output_dir output_tta \
+ --export_model_dir output_tta \
+ --per_device_train_batch_size 8 \
+ --save_steps 1 \
+ --eval_steps 1 \
+ --max_steps 2 \
+ --overwrite_output_dir \
+ --remove_unused_columns False >${log_path}/ernie-m_tta >>${log_path}/ernie-m_tta 2>&1
+ print_info $? ernie-m_tta
+ else
+ python -m pytest ${nlp_dir}/tests/model_zoo/test_ernie_m.py >${log_path}/ernie-m >>${log_path}/ernie-m 2>&1
+ print_info $? ernie-m
+ fi
+}
+#32 clue
+clue() {
+ cd ${nlp_dir}/legacy/examples/benchmark/clue/classification
+ python -u ./run_clue_classifier_trainer.py \
+ --model_name_or_path ernie-3.0-base-zh \
+ --dataset "clue afqmc" \
+ --max_seq_length 128 \
+ --per_device_train_batch_size 32 \
+ --per_device_eval_batch_size 32 \
+ --learning_rate 1e-5 \
+ --num_train_epochs 3 \
+ --logging_steps 1 \
+ --seed 42 \
+ --save_steps 3 \
+ --warmup_ratio 0.1 \
+ --weight_decay 0.01 \
+ --adam_epsilon 1e-8 \
+ --output_dir ./tmp \
--device gpu \
- --task_type cross-lingual-transfer \
- --model_name_or_path __internal_testing__/ernie-m \
- --use_test_data True \
- --test_data_path ../../tests/fixtures/tests_samples/xnli/xnli.jsonl \
- --output_dir output_clt \
- --export_model_dir output_clt \
- --per_device_train_batch_size 8 \
- --save_steps 1 \
- --eval_steps 1 \
- --max_steps 2 \
- --overwrite_output_dir \
- --remove_unused_columns False >${log_path}/ernie-m_clt >>${log_path}/ernie-m_clt 2>&1
- print_info $? ernie-m_clt
- # finetuned for translate-train-all
- python -m paddle.distributed.launch --log_dir output_tta run_classifier.py \
--do_train \
--do_eval \
- --do_export \
- --device gpu \
- --task_type translate-train-all \
- --model_name_or_path __internal_testing__/ernie-m \
- --use_test_data True \
- --test_data_path ../../tests/fixtures/tests_samples/xnli/xnli.jsonl \
- --output_dir output_tta \
- --export_model_dir output_tta \
- --per_device_train_batch_size 8 \
+ --metric_for_best_model "eval_accuracy" \
+ --load_best_model_at_end \
+ --save_total_limit 1 \
+ --max_steps 1 >${log_path}/clue-trainer_api >>${log_path}/clue-trainer_api 2>&1
+ print_info $? clue-tranier_api
+ python -u run_clue_classifier.py \
+ --model_name_or_path ernie-3.0-base-zh \
+ --task_name afqmc \
+ --max_seq_length 128 \
+ --batch_size 16 \
+ --learning_rate 3e-5 \
+ --num_train_epochs 3 \
+ --logging_steps 100 \
+ --seed 42 \
--save_steps 1 \
- --eval_steps 1 \
- --max_steps 2 \
- --overwrite_output_dir \
- --remove_unused_columns False >${log_path}/ernie-m_tta >>${log_path}/ernie-m_tta 2>&1
- print_info $? ernie-m_tta
-else
- python -m pytest ${nlp_dir}/tests/model_zoo/test_ernie_m.py >${log_path}/ernie-m >>${log_path}/ernie-m 2>&1
- print_info $? ernie-m
-fi
-}
-#32 clue
-clue (){
-cd ${nlp_dir}/examples/benchmark/clue/classification
-python -u ./run_clue_classifier_trainer.py \
- --model_name_or_path ernie-3.0-base-zh \
- --dataset "clue afqmc" \
- --max_seq_length 128 \
- --per_device_train_batch_size 32 \
- --per_device_eval_batch_size 32 \
- --learning_rate 1e-5 \
- --num_train_epochs 3 \
- --logging_steps 1 \
- --seed 42 \
- --save_steps 3 \
- --warmup_ratio 0.1 \
- --weight_decay 0.01 \
- --adam_epsilon 1e-8 \
- --output_dir ./tmp \
- --device gpu \
- --do_train \
- --do_eval \
- --metric_for_best_model "eval_accuracy" \
- --load_best_model_at_end \
- --save_total_limit 1 \
- --max_steps 1 >${log_path}/clue-trainer_api >>${log_path}/clue-trainer_api 2>&1
-print_info $? clue-tranier_api
-python -u run_clue_classifier.py \
- --model_name_or_path ernie-3.0-base-zh \
- --task_name afqmc \
- --max_seq_length 128 \
- --batch_size 16 \
- --learning_rate 3e-5 \
- --num_train_epochs 3 \
- --logging_steps 100 \
- --seed 42 \
- --save_steps 1 \
- --warmup_proportion 0.1 \
- --weight_decay 0.01 \
- --adam_epsilon 1e-8 \
- --output_dir ./output/afqmc \
- --device gpu \
- --max_steps 1 \
- --do_train >${log_path}/clue-class >>${log_path}/clue-class 2>&1
-print_info $? clue-class
-cd ${nlp_dir}/examples/benchmark/clue/mrc
-export CUDA_VISIBLE_DEVICES=${cudaid1}
-python -m paddle.distributed.launch run_cmrc2018.py \
- --model_name_or_path ernie-3.0-base-zh \
- --batch_size 16 \
- --learning_rate 3e-5 \
- --max_seq_length 512 \
- --num_train_epochs 2 \
- --do_train \
- --do_predict \
- --warmup_proportion 0.1 \
- --weight_decay 0.01 \
- --gradient_accumulation_steps 2 \
- --max_steps 1 \
- --output_dir ./tmp >${log_path}/clue-mrc >>${log_path}/clue-mrc 2>&1
-print_info $? clue-mrc
-}
-#32 textcnn
-textcnn(){
-cd ${nlp_dir}/examples/sentiment_analysis/textcnn
-wget https://bj.bcebos.com/paddlenlp/datasets/RobotChat.tar.gz
-tar xvf RobotChat.tar.gz
-wget https://bj.bcebos.com/paddlenlp/robot_chat_word_dict.txt
-wget https://bj.bcebos.com/paddlenlp/models/textcnn.pdparams
-python -m paddle.distributed.launch train.py \
- --vocab_path=./robot_chat_word_dict.txt \
- --init_from_ckpt=./textcnn.pdparams \
- --device=gpu \
- --lr=5e-5 \
- --batch_size=64 \
- --epochs=1 \
- --save_dir=./checkpoints \
- --data_path=./RobotChat >${log_path}/textcnn_train >>${log_path}/textcnn_train 2>&1
-print_info $? textcnn_train
-python export_model.py --vocab_path=./robot_chat_word_dict.txt --params_path=./checkpoints/final.pdparams \
- --output_path=./static_graph_params >${log_path}/textcnn_export >>${log_path}/textcnn_export 2>&1
-print_info $? export_export
-python deploy/python/predict.py --model_file=static_graph_params.pdmodel \
- --params_file=static_graph_params.pdiparams >${log_path}/textcnn_depoly >>${log_path}/textcnn_depoly 2>&1
-print_info $? textcnn_deploy
-python predict.py --vocab_path=./robot_chat_word_dict.txt \
- --device=gpu \
- --params_path=./checkpoints/final.pdparams >${log_path}/textcnn_predict >>${log_path}/textcnn_predict 2>&1
-print_info $? textcnn_predict
+ --warmup_proportion 0.1 \
+ --weight_decay 0.01 \
+ --adam_epsilon 1e-8 \
+ --output_dir ./output/afqmc \
+ --device gpu \
+ --max_steps 1 \
+ --do_train >${log_path}/clue-class >>${log_path}/clue-class 2>&1
+ print_info $? clue-class
+ cd ${nlp_dir}/examples/benchmark/clue/mrc
+ export CUDA_VISIBLE_DEVICES=${cudaid1}
+ python -m paddle.distributed.launch run_cmrc2018.py \
+ --model_name_or_path ernie-3.0-base-zh \
+ --batch_size 16 \
+ --learning_rate 3e-5 \
+ --max_seq_length 512 \
+ --num_train_epochs 2 \
+ --do_train \
+ --do_predict \
+ --warmup_proportion 0.1 \
+ --weight_decay 0.01 \
+ --gradient_accumulation_steps 2 \
+ --max_steps 1 \
+ --output_dir ./tmp >${log_path}/clue-mrc >>${log_path}/clue-mrc 2>&1
+ print_info $? clue-mrc
}
#33 taskflow
taskflow (){
-cd ${nlp_dir}
-python -m pytest tests/taskflow/test_*.py >${nlp_dir}/unittest_logs/taskflow_unittest >>${nlp_dir}/unittest_logs/taskflow_unittest 2>&1
-print_info $? taskflow_unittest
-python -m pytest scripts/regression/test_taskflow.py >${log_path}/taskflow >>${log_path}/taskflow 2>&1
-print_info $? taskflow
+ cd ${nlp_dir}
+ python -m pytest tests/taskflow/test_*.py >${nlp_dir}/unittest_logs/taskflow_unittest >>${nlp_dir}/unittest_logs/taskflow_unittest 2>&1
+ print_info $? taskflow_unittest
+ python -m pytest scripts/regression/test_taskflow.py >${log_path}/taskflow >>${log_path}/taskflow 2>&1
+ print_info $? taskflow
}
llm(){
-cd ${nlp_dir}/csrc
-echo "build paddlenlp_op"
-python setup_cuda.py install
+ cd ${nlp_dir}/csrc
+ echo "build paddlenlp_op"
+ python setup_cuda.py install
-echo ' Testing all LLMs '
-cd ${nlp_dir}
-python -m pytest tests/llm/test_*.py --alluredir=result >${log_path}/llm >>${log_path}/llm 2>&1
-print_info $? llm
+ echo ' Testing all LLMs '
+ cd ${nlp_dir}
+ python -m pytest tests/llm/test_*.py --alluredir=result >${log_path}/llm >>${log_path}/llm 2>&1
+ print_info $? llm
}
fast_generation(){
-cd ${nlp_dir}/fast_generation/samples
-# python codegen_sample.py >${log_path}/fast_generation_codegen >>${log_path}/fast_generation_codegen 2>&1
-# print_info $? fast_generation_codegen
+ cd ${nlp_dir}/fast_generation/samples
+ # python codegen_sample.py >${log_path}/fast_generation_codegen >>${log_path}/fast_generation_codegen 2>&1
+ # print_info $? fast_generation_codegen
-python gpt_sample.py >${log_path}/fast_generation_gpt >>${log_path}/fast_generation_gpt 2>&1
-print_info $? fast_generation_gpt
+ python gpt_sample.py >${log_path}/fast_generation_gpt >>${log_path}/fast_generation_gpt 2>&1
+ print_info $? fast_generation_gpt
-python mbart_sample.py >${log_path}/fast_generation_mbart >>${log_path}/fast_generation_mbart 2>&1
-print_info $? fast_generation_mbart
+ python mbart_sample.py >${log_path}/fast_generation_mbart >>${log_path}/fast_generation_mbart 2>&1
+ print_info $? fast_generation_mbart
-python plato_sample.py >${log_path}/fast_generation_plato >>${log_path}/fast_generation_plato 2>&1
-print_info $? fast_generation_plato
+ python plato_sample.py >${log_path}/fast_generation_plato >>${log_path}/fast_generation_plato 2>&1
+ print_info $? fast_generation_plato
-python t5_sample.py --use_faster >${log_path}/fast_generation_t5 >>${log_path}/fast_generation_t5 2>&1
-print_info $? fast_generation_t5
+ python t5_sample.py --use_faster >${log_path}/fast_generation_t5 >>${log_path}/fast_generation_t5 2>&1
+ print_info $? fast_generation_t5
-cd ${nlp_dir}/paddlenlp/ops/fast_transformer/sample/
-python bart_decoding_sample.py >${log_path}/fast_generation_bart >>${log_path}/fast_generation_bart 2>&1
-print_info $? fast_generation_bart
+ cd ${nlp_dir}/paddlenlp/ops/fast_transformer/sample/
+ python bart_decoding_sample.py >${log_path}/fast_generation_bart >>${log_path}/fast_generation_bart 2>&1
+ print_info $? fast_generation_bart
-python t5_export_model_sample.py >${log_path}/t5_export_model_sample >>${log_path}/t5_export_model_sample 2>&1
-print_info $? t5_export_model_sample
+ python t5_export_model_sample.py >${log_path}/t5_export_model_sample >>${log_path}/t5_export_model_sample 2>&1
+ print_info $? t5_export_model_sample
-python t5_export_model_sample.py >${log_path}/t5_export_model_sample >>${log_path}/t5_export_model_sample 2>&1
-print_info $? t5_export_model_sample
+ python t5_export_model_sample.py >${log_path}/t5_export_model_sample >>${log_path}/t5_export_model_sample 2>&1
+ print_info $? t5_export_model_sample
-# fast_gpt
-# fast_transformer
+ # fast_gpt
+ # fast_transformer
}
ernie-3.0(){
-cd ${nlp_dir}/model_zoo/ernie-3.0/
-#训练
-python run_seq_cls.py --model_name_or_path ernie-3.0-medium-zh --dataset afqmc --output_dir ./best_models --export_model_dir best_models/ --do_train --do_eval --do_export --config=configs/default.yml --max_steps=2 --save_step=2 >${log_path}/ernie-3.0_train_seq_cls >>${log_path}/ernie-3.0_train_seq_cls 2>&1
-print_info $? ernie-3.0_train_seq_cls
-python run_token_cls.py --model_name_or_path ernie-3.0-medium-zh --dataset msra_ner --output_dir ./best_models --export_model_dir best_models/ --do_train --do_eval --do_export --config=configs/default.yml --max_steps=2 --save_step=2 >${log_path}/ernie-3.0_train_token_cls >>${log_path}/ernie-3.0_train_token_cls 2>&1
-print_info $? ernie-3.0_train_token_cls
-python run_qa.py --model_name_or_path ernie-3.0-medium-zh --dataset cmrc2018 --output_dir ./best_models --export_model_dir best_models/ --do_train --do_eval --do_export --config=configs/default.yml --max_steps=2 --save_step=2 >${log_path}/ernie-3.0_train_qa >>${log_path}/ernie-3.0_train_qa 2>&1
-print_info $? ernie-3.0_train_qa
-# 预测
-python run_seq_cls.py --model_name_or_path best_models/afqmc/ --dataset afqmc --output_dir ./best_models --do_predict --config=configs/default.yml >${log_path}/ernie-3.0_predict_seq_cls >>${log_path}/ernie-3.0_predict_seq_cls 2>&1
-print_info $? ernie-3.0_predict_seq_cls
-python run_token_cls.py --model_name_or_path best_models/msra_ner/ --dataset msra_ner --output_dir ./best_models --do_predict --config=configs/default.yml >${log_path}/ernie-3.0_predict_token_cls >>${log_path}/ernie-3.0_predict_token_cls 2>&1
-print_info $? ernie-3.0_predict_token_cls
-python run_qa.py --model_name_or_path best_models/cmrc2018/ --dataset cmrc2018 --output_dir ./best_models --do_predict --config=configs/default.yml >${log_path}/ernie-3.0_predict_qa >>${log_path}/ernie-3.0_predict_qa 2>&1
-print_info $? ernie-3.0_predict_qa
-#压缩
-python compress_seq_cls.py --model_name_or_path best_models/afqmc/ --dataset afqmc --output_dir ./best_models/afqmc --config=configs/default.yml --max_steps 10 --eval_steps 5 --save_steps 5 --save_steps 5 --algo_list mse --batch_size_list 4 >${log_path}/ernie-3.0_compress_seq_cls >>${log_path}/ernie-3.0_compress_seq_cls 2>&1
-print_info $? ernie-3.0_compress_seq_cls
-python compress_token_cls.py --model_name_or_path best_models/msra_ner/ --dataset msra_ner --output_dir ./best_models/msra_ner --config=configs/default.yml --max_steps 10 --eval_steps 5 --save_steps 5 --algo_list mse --batch_size_list 4 >${log_path}/ernie-3.0_compress_token_cls >>${log_path}/ernie-3.0_compress_token_cls 2>&1
-print_info $? ernie-3.0_compress_token_cls
-python compress_qa.py --model_name_or_path best_models/cmrc2018/ --dataset cmrc2018 --output_dir ./best_models/cmrc2018 --config=configs/default.yml --max_steps 10 --eval_steps 5 --save_steps 5 --algo_list mse --batch_size_list 4 >${log_path}/ernie-3.0_compress_qa >>${log_path}/ernie-3.0_compress_qa 2>&1
-print_info $? ernie-3.0_compress_qa
+ cd ${nlp_dir}/model_zoo/ernie-3.0/
+ #训练
+ python run_seq_cls.py --model_name_or_path ernie-3.0-medium-zh --dataset afqmc --output_dir ./best_models --export_model_dir best_models/ --do_train --do_eval --do_export --config=configs/default.yml --max_steps=2 --save_step=2 >${log_path}/ernie-3.0_train_seq_cls >>${log_path}/ernie-3.0_train_seq_cls 2>&1
+ print_info $? ernie-3.0_train_seq_cls
+ python run_token_cls.py --model_name_or_path ernie-3.0-medium-zh --dataset msra_ner --output_dir ./best_models --export_model_dir best_models/ --do_train --do_eval --do_export --config=configs/default.yml --max_steps=2 --save_step=2 >${log_path}/ernie-3.0_train_token_cls >>${log_path}/ernie-3.0_train_token_cls 2>&1
+ print_info $? ernie-3.0_train_token_cls
+ python run_qa.py --model_name_or_path ernie-3.0-medium-zh --dataset cmrc2018 --output_dir ./best_models --export_model_dir best_models/ --do_train --do_eval --do_export --config=configs/default.yml --max_steps=2 --save_step=2 >${log_path}/ernie-3.0_train_qa >>${log_path}/ernie-3.0_train_qa 2>&1
+ print_info $? ernie-3.0_train_qa
+ # 预测
+ python run_seq_cls.py --model_name_or_path best_models/afqmc/ --dataset afqmc --output_dir ./best_models --do_predict --config=configs/default.yml >${log_path}/ernie-3.0_predict_seq_cls >>${log_path}/ernie-3.0_predict_seq_cls 2>&1
+ print_info $? ernie-3.0_predict_seq_cls
+ python run_token_cls.py --model_name_or_path best_models/msra_ner/ --dataset msra_ner --output_dir ./best_models --do_predict --config=configs/default.yml >${log_path}/ernie-3.0_predict_token_cls >>${log_path}/ernie-3.0_predict_token_cls 2>&1
+ print_info $? ernie-3.0_predict_token_cls
+ python run_qa.py --model_name_or_path best_models/cmrc2018/ --dataset cmrc2018 --output_dir ./best_models --do_predict --config=configs/default.yml >${log_path}/ernie-3.0_predict_qa >>${log_path}/ernie-3.0_predict_qa 2>&1
+ print_info $? ernie-3.0_predict_qa
+ #压缩
+ python compress_seq_cls.py --model_name_or_path best_models/afqmc/ --dataset afqmc --output_dir ./best_models/afqmc --config=configs/default.yml --max_steps 10 --eval_steps 5 --save_steps 5 --save_steps 5 --algo_list mse --batch_size_list 4 >${log_path}/ernie-3.0_compress_seq_cls >>${log_path}/ernie-3.0_compress_seq_cls 2>&1
+ print_info $? ernie-3.0_compress_seq_cls
+ python compress_token_cls.py --model_name_or_path best_models/msra_ner/ --dataset msra_ner --output_dir ./best_models/msra_ner --config=configs/default.yml --max_steps 10 --eval_steps 5 --save_steps 5 --algo_list mse --batch_size_list 4 >${log_path}/ernie-3.0_compress_token_cls >>${log_path}/ernie-3.0_compress_token_cls 2>&1
+ print_info $? ernie-3.0_compress_token_cls
+ python compress_qa.py --model_name_or_path best_models/cmrc2018/ --dataset cmrc2018 --output_dir ./best_models/cmrc2018 --config=configs/default.yml --max_steps 10 --eval_steps 5 --save_steps 5 --algo_list mse --batch_size_list 4 >${log_path}/ernie-3.0_compress_qa >>${log_path}/ernie-3.0_compress_qa 2>&1
+ print_info $? ernie-3.0_compress_qa
}
ernie-health(){
-cd ${nlp_dir}/tests/model_zoo/
-if [ ! -f 'test_ernie-health.py' ];then
- echo '模型测试文件不存在!'
-else
- python -m pytest tests/model_zoo/test_ernie-health.py >${log_path}/ernie-health_unittest>>${log_path}/ernie-health_unittest 2>&1
- print_info $? tests ernie-health_unittest
-fi
+ cd ${nlp_dir}/tests/model_zoo/
+ if [ ! -f 'test_ernie-health.py' ];then
+ echo '模型测试文件不存在!'
+ else
+ python -m pytest tests/model_zoo/test_ernie-health.py >${log_path}/ernie-health_unittest>>${log_path}/ernie-health_unittest 2>&1
+ print_info $? tests ernie-health_unittest
+ fi
}
uie(){
-cd ${nlp_dir}/model_zoo/uie/
-mkdir data && cd data && wget https://bj.bcebos.com/paddlenlp/datasets/uie/doccano_ext.json && cd ../
-python doccano.py --doccano_file ./data/doccano_ext.json --task_type ext --save_dir ./data --splits 0.8 0.2 0 --schema_lang ch >${log_path}/uie_doccano>>${log_path}/uie_doccano 2>&1
-print_info $? uie_doccano
-python -u -m paddle.distributed.launch finetune.py --device gpu --logging_steps 2 --save_steps 2 --eval_steps 2 --seed 42 \
- --model_name_or_path uie-base --output_dir ./checkpoint/model_best --train_path data/train.txt --dev_path data/dev.txt \
- --max_seq_length 512 --per_device_eval_batch_size 16 --per_device_train_batch_size 16 --num_train_epochs 100 --learning_rate 1e-5 \
- --do_train --do_eval --do_export --export_model_dir ./checkpoint/model_best --label_names start_positions end_positions \
- --overwrite_output_dir --disable_tqdm True --metric_for_best_model eval_f1 --load_best_model_at_end True \
- --save_total_limit 1 --max_steps 2 >${log_path}/uie_train>>${log_path}/uie_train2>&1
-print_info $? uie_train
-python evaluate.py --model_path ./checkpoint/model_best --test_path ./data/dev.txt --batch_size 16 --max_seq_len 512 >${log_path}/uie_eval>>${log_path}/uie_eval 2>&1
-print_info $? uie_eval
+ cd ${nlp_dir}/model_zoo/uie/
+ mkdir data && cd data && wget https://bj.bcebos.com/paddlenlp/datasets/uie/doccano_ext.json && cd ../
+ python doccano.py --doccano_file ./data/doccano_ext.json --task_type ext --save_dir ./data --splits 0.8 0.2 0 --schema_lang ch >${log_path}/uie_doccano>>${log_path}/uie_doccano 2>&1
+ print_info $? uie_doccano
+ python -u -m paddle.distributed.launch finetune.py --device gpu --logging_steps 2 --save_steps 2 --eval_steps 2 --seed 42 \
+ --model_name_or_path uie-base --output_dir ./checkpoint/model_best --train_path data/train.txt --dev_path data/dev.txt \
+ --max_seq_length 512 --per_device_eval_batch_size 16 --per_device_train_batch_size 16 --num_train_epochs 100 --learning_rate 1e-5 \
+ --do_train --do_eval --do_export --export_model_dir ./checkpoint/model_best --label_names start_positions end_positions \
+ --overwrite_output_dir --disable_tqdm True --metric_for_best_model eval_f1 --load_best_model_at_end True \
+ --save_total_limit 1 --max_steps 2 >${log_path}/uie_train>>${log_path}/uie_train2>&1
+ print_info $? uie_train
+ python evaluate.py --model_path ./checkpoint/model_best --test_path ./data/dev.txt --batch_size 16 --max_seq_len 512 >${log_path}/uie_eval>>${log_path}/uie_eval 2>&1
+ print_info $? uie_eval
}
ernie-layout(){
-cd ${nlp_dir}/model_zoo/ernie-layout/
-# train ner
-python -u run_ner.py --model_name_or_path ernie-layoutx-base-uncased --output_dir ./ernie-layoutx-base-uncased/models/funsd/ \
- --dataset_name funsd --do_train --do_eval --max_steps 2 --eval_steps 2 --save_steps 2 --save_total_limit 1 --seed 1000 --overwrite_output_dir \
- --load_best_model_at_end --pattern ner-bio --preprocessing_num_workers 4 --overwrite_cache false --doc_stride 128 --target_size 1000 \
- --per_device_train_batch_size 4 --per_device_eval_batch_size 4 --learning_rate 2e-5 --lr_scheduler_type constant --gradient_accumulation_steps 1 \
- --metric_for_best_model eval_f1 --greater_is_better true >${log_path}/ernie-layout_train>>${log_path}/ernie-layout_train 2>&1
-print_info $? ernie-layout_train
-# export ner
-python export_model.py --task_type ner --model_path ./ernie-layoutx-base-uncased/models/funsd/ --output_path ./ner_export >${log_path}/ernie-layout_export>>${log_path}/ernie-layout_export2>&1
-print_info $? ernie-layout_export
-# deploy ner
-cd ${nlp_dir}/model_zoo/ernie-layout/deploy/python
-wget https://bj.bcebos.com/paddlenlp/datasets/document_intelligence/images.zip && unzip images.zip
-python infer.py --model_path_prefix ../../ner_export/inference --task_type ner --lang "en" --batch_size 8 >${log_path}/ernie-layout_deploy>>${log_path}/ernie-layout_deploy 2>&1
-print_info $? ernie-layout_deploy
+ cd ${nlp_dir}/model_zoo/ernie-layout/
+ # train ner
+ python -u run_ner.py --model_name_or_path ernie-layoutx-base-uncased --output_dir ./ernie-layoutx-base-uncased/models/funsd/ \
+ --dataset_name funsd --do_train --do_eval --max_steps 2 --eval_steps 2 --save_steps 2 --save_total_limit 1 --seed 1000 --overwrite_output_dir \
+ --load_best_model_at_end --pattern ner-bio --preprocessing_num_workers 4 --overwrite_cache false --doc_stride 128 --target_size 1000 \
+ --per_device_train_batch_size 4 --per_device_eval_batch_size 4 --learning_rate 2e-5 --lr_scheduler_type constant --gradient_accumulation_steps 1 \
+ --metric_for_best_model eval_f1 --greater_is_better true >${log_path}/ernie-layout_train>>${log_path}/ernie-layout_train 2>&1
+ print_info $? ernie-layout_train
+ # export ner
+ python export_model.py --task_type ner --model_path ./ernie-layoutx-base-uncased/models/funsd/ --output_path ./ner_export >${log_path}/ernie-layout_export>>${log_path}/ernie-layout_export2>&1
+ print_info $? ernie-layout_export
+ # deploy ner
+ cd ${nlp_dir}/model_zoo/ernie-layout/deploy/python
+ wget https://bj.bcebos.com/paddlenlp/datasets/document_intelligence/images.zip && unzip images.zip
+ python infer.py --model_path_prefix ../../ner_export/inference --task_type ner --lang "en" --batch_size 8 >${log_path}/ernie-layout_deploy>>${log_path}/ernie-layout_deploy 2>&1
+ print_info $? ernie-layout_deploy
}
ernie-1.0(){
ernie
@@ -1082,17 +898,13 @@ ernie_m(){
}
ernie_layout(){
-ernie-layout
+ ernie-layout
}
ernie_csc(){
ernie-csc
}
-ernie_ctm(){
- ernie-ctm
-}
-
ernie_doc(){
ernie-doc
}
@@ -1102,25 +914,25 @@ ernie_health(){
}
segment_parallel_utils(){
-cd ${nlp_dir}
-echo "test segment_parallel_utils, cudaid1:${cudaid1}, cudaid2:${cudaid2}"
-if [[ ${cudaid1} != ${cudaid2} ]]; then
- time (python -m paddle.distributed.launch tests/transformers/test_segment_parallel_utils.py >${log_path}/segment_parallel_utils) >>${log_path}/segment_parallel_utils 2>&1
- print_info $? segment_parallel_utils
-else
- echo "only one gpu:${cudaid1} is set, skip test"
-fi
+ cd ${nlp_dir}
+ echo "test segment_parallel_utils, cudaid1:${cudaid1}, cudaid2:${cudaid2}"
+ if [[ ${cudaid1} != ${cudaid2} ]]; then
+ time (python -m paddle.distributed.launch tests/transformers/test_segment_parallel_utils.py >${log_path}/segment_parallel_utils) >>${log_path}/segment_parallel_utils 2>&1
+ print_info $? segment_parallel_utils
+ else
+ echo "only one gpu:${cudaid1} is set, skip test"
+ fi
}
ring_flash_attention(){
-cd ${nlp_dir}
-echo "test ring_flash_attention, cudaid1:${cudaid1}, cudaid2:${cudaid2}"
-if [[ ${cudaid1} != ${cudaid2} ]]; then
- time (python -m paddle.distributed.launch tests/transformers/test_ring_flash_attention.py >${log_path}/ring_flash_attention) >>${log_path}/ring_flash_attention 2>&1
- print_info $? ring_flash_attention
-else
- echo "only one gpu:${cudaid1} is set, skip test"
-fi
+ cd ${nlp_dir}
+ echo "test ring_flash_attention, cudaid1:${cudaid1}, cudaid2:${cudaid2}"
+ if [[ ${cudaid1} != ${cudaid2} ]]; then
+ time (python -m paddle.distributed.launch tests/transformers/test_ring_flash_attention.py >${log_path}/ring_flash_attention) >>${log_path}/ring_flash_attention 2>&1
+ print_info $? ring_flash_attention
+ else
+ echo "only one gpu:${cudaid1} is set, skip test"
+ fi
}
$1
diff --git a/scripts/regression/get_model_list.py b/scripts/regression/get_model_list.py
index 97bfff2c3b5d..3e90ea2bb404 100644
--- a/scripts/regression/get_model_list.py
+++ b/scripts/regression/get_model_list.py
@@ -20,97 +20,68 @@ def get_model_list():
"""
get model list from
-
+
"""
CI_MODEL_LIST = [
- "waybill_ie",
- "msra_ner",
- "glue",
+ "DuEE",
+ "DuReader-robust",
+ "DuReader-yesno",
+ "SQuAD",
+ "albert",
"bert",
- "skep",
"bigbird",
+ "clue",
+ "couplet",
+ "doc",
"electra",
- "gpt",
+ "elmo",
"ernie",
- "xlnet",
- "ofa",
- "albert",
- "squad",
- "tinybert",
- "lexical_analysis",
- "seq2seq",
- "pretrained_models",
- "word_embedding",
- "ernie-ctm",
- "distilbert",
- "stacl",
- "transformer",
- "simbert",
+ "ernie-1.0",
+ "ernie-csc",
"ernie-doc",
- "transformer-xl",
+ "ernie-gen",
+ "ernie-health",
"ernie-m",
- "plato-xl",
- "pointer_summarizer",
- "question_matching",
+ "ernie_matching",
"few_shot",
- "unimo-text",
- "ernie-csc",
- "nptag",
- "ofa",
- "transformer",
- "DuIE",
- "tcn",
- "word_embedding",
- "unified_transformer",
- "lic2021_baseline",
- "vae-seq2seq",
+ "glue",
+ "gpt",
+ "gpt-3",
+ "lexical_analysis",
+ "minilmv2",
+ "mpnet",
"msra_ner",
- "simbert",
- "clue",
- "pet",
- "bert",
- "ernie-ctm",
- "DuReader-yesno",
- "nptag",
- "semantic_indexing",
- "seq2seq",
+ "msra_ner",
+ "ofa",
"pointer_summarizer",
- "bigbird",
- "unimo-text",
- "minilmv2",
- "wordtag",
- "simcse",
- "ernie-gen",
- "distill_lstm",
- "DuReader-robust",
- "ernie_matching",
- "rnn",
- "ernie-1.0",
- "stacl",
- "erniesage",
- "DuEE",
- "efl",
- "doc",
- "couplet",
- "rnnlm",
"pp-minilm",
- "dgu",
- "mpnet",
- "textcnn",
- "p-tuning",
- "SQuAD",
- "elmo",
- "plato-2",
"pretrained_models",
+ "question_matching",
+ "rnn",
+ "rnnlm",
+ "semantic_indexing",
"sentiment_analysis",
- "ernie-health",
- "gpt-3",
+ "simbert",
+ "simbert",
+ "simcse",
+ "skep",
+ "squad",
+ "stacl",
+ "stacl",
+ "tcn",
+ "tinybert",
+ "transformer",
+ "transformer-xl",
+ "unimo-text",
+ "vae-seq2seq",
+ "word_embedding",
+ "xlnet",
]
examples_second_list = ["model_interpretation", "semantic_indexing", "lexical_analysis", "word_embedding"]
model_list = os.listdir("model_zoo")
- examples_list = os.listdir("examples/")
+ examples_list = os.listdir("legacy/examples/")
app_list = os.listdir("applications/")
# remove model_list README
diff --git a/scripts/regression/run_ci.sh b/scripts/regression/run_ci.sh
index af2e164947e0..d4490304b7ee 100644
--- a/scripts/regression/run_ci.sh
+++ b/scripts/regression/run_ci.sh
@@ -28,12 +28,12 @@ export APIcase_list=()
declare -A Normal_dic
declare -A all_P0case_dic
declare -A Build_list
-all_P0case_dic=(["waybill_ie"]=3 ["msra_ner"]=15 ["glue"]=2 ["bert"]=2 ["skep"]=10 ["bigbird"]=2 ["electra"]=2 ["gpt"]=2 ["ernie-1.0"]=2 ["xlnet"]=2 \
-["ofa"]=2 ["albert"]=2 ["SQuAD"]=20 ["lexical_analysis"]=5 ["seq2seq"]=5 ["word_embedding"]=5 \
-["ernie-ctm"]=5 ["distilbert"]=5 ["transformer"]=5 ["pet"]=5 ["efl"]=5 ["p-tuning"]=5 ["ernie-doc"]=20 ["transformer-xl"]=5 \
-["question_matching"]=5 ["ernie-csc"]=5 ["nptag"]=5 ["ernie-m"]=5 ["taskflow"]=5 ["clue"]=5 ["textcnn"]=5 \
-["fast_generation"]=10 ["ernie-3.0"]=5 ["ernie-layout"]=5 ["uie"]=5 ["ernie-health"]=5 ["llm"]=5 \
-["ernie"]=2 ["ernie_m"]=5 ["ernie_layout"]=5 ["ernie_csc"]=5 ["ernie_ctm"]=5 ["ernie_doc"]=20 ["ernie_health"]=5 ["segment_parallel_utils"]=5 ["ring_flash_attention"]=5)
+all_P0case_dic=("msra_ner"]=15 ["glue"]=2 ["bert"]=2 ["skep"]=10 ["bigbird"]=2 ["electra"]=2 ["gpt"]=2 ["ernie-1.0"]=2 ["xlnet"]=2
+ ["ofa"]=2 ["albert"]=2 ["SQuAD"]=20 ["lexical_analysis"]=5 ["word_embedding"]=5
+ ["transformer"]=5 ["ernie-doc"]=20 ["transformer-xl"]=5
+ ["question_matching"]=5 ["ernie-csc"]=5 ["ernie-m"]=5 ["taskflow"]=5 ["clue"]=5 ["textcnn"]=5
+ ["fast_generation"]=10 ["ernie-3.0"]=5 ["ernie-layout"]=5 ["uie"]=5 ["ernie-health"]=5 ["llm"]=5
+ ["ernie"]=2 ["ernie_m"]=5 ["ernie_layout"]=5 ["ernie_csc"]=5 ["ernie_ctm"]=5 ["ernie_doc"]=20 ["ernie_health"]=5 ["segment_parallel_utils"]=5 ["ring_flash_attention"]=5)
####################################
python -m pip config --user set global.index http://pip.baidu-int.com/search/
diff --git a/scripts/regression/run_release.sh b/scripts/regression/run_release.sh
index 6aa02bbd8dff..354960ea7fe0 100644
--- a/scripts/regression/run_release.sh
+++ b/scripts/regression/run_release.sh
@@ -54,7 +54,7 @@ export all_P0case_time=0
declare -A all_P0case_dic
get_diff_TO_P0case(){
if [[ ${Testcase} =~ "all" ]];then
- P0case_list=(waybill_ie msra_ner glue bert skep bigbird electra gpt ernie-1.0 xlnet ofa squad tinybert lexical_analysis seq2seq \
+ P0case_list=(msra_ner glue bert skep bigbird electra gpt ernie-1.0 xlnet ofa squad tinybert lexical_analysis seq2seq \
word_embedding ernie-ctm distilbert stacl transformer simbert ernie-doc transformer-xl pointer_summarizer question_matching ernie-csc \
nptag ernie-m clue taskflow transformers fast_generation ernie-3.0 fast_transformer fast_gpt llama)
elif [[ ${Testcase} =~ "p0" ]];then