From b6ac25e7f050545bc99bcf7a5b28da69a3e2f971 Mon Sep 17 00:00:00 2001 From: iosmers Date: Wed, 10 Jan 2024 14:29:37 +0800 Subject: [PATCH 01/17] add vpp ce --- ...qlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh | 3 ++- ...train_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh | 6 +++--- ...train_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh index 6803f8b4416a..94672f9090a3 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh @@ -31,8 +31,9 @@ sequence_parallel=True model=gpt micro_bs=8 +seed=3589 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} 2>&1; \ No newline at end of file +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh index e476037f9ada..c5b31056a667 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh @@ -18,7 +18,7 @@ mp_degree=2 pp_degree=4 bs_item=32 fp_item=bf16 -run_mode=MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8 +run_mode=MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2 device_num=N2C16 max_iter=50000 sharding=stage1 @@ -30,8 +30,8 @@ use_pipeline_parallel=True sequence_parallel=True model=gpt -micro_bs=2 -acc=8 +micro_bs=8 +acc=2 seed=3589 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh index 03a50a587e9e..c1e68d45b7d4 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh @@ -18,7 +18,7 @@ mp_degree=2 pp_degree=4 bs_item=32 fp_item=bf16 -run_mode=MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8 +run_mode=MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2 device_num=N2C16 max_iter=100 sharding=stage1 @@ -31,8 +31,8 @@ use_pipeline_parallel=True sequence_parallel=True model=gpt -micro_bs=2 -acc=8 +micro_bs=8 +acc=2 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run From bedbd99c67ca5eb07b7988d37adef3a6acb55059 Mon Sep 17 00:00:00 2001 From: iosmers Date: Wed, 10 Jan 2024 14:49:59 +0800 Subject: [PATCH 02/17] add test --- ...ain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh | 2 +- ...in_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh | 41 +++++++++++++++++++ ...s32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh | 41 +++++++++++++++++++ ...6_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh | 2 +- ...in_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh | 41 +++++++++++++++++++ ...s32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh | 41 +++++++++++++++++++ ...6_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh | 3 +- 7 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh index 94672f9090a3..a19e0034f87a 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh @@ -31,7 +31,7 @@ sequence_parallel=True model=gpt micro_bs=8 -seed=3589 +seed=1234 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh new file mode 100644 index 000000000000..bd0308d493fb --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=CE_gpt-345m_seqlen1024_pretrain +dp_degree=1 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-PP4-VPP2-DP2-mbs8-acc2 +device_num=N2C16 +max_iter=50000 +sharding=False +sharding_degree=1 + +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=False + +model=gpt +micro_bs=8 +acc=2 +seed=1234 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh new file mode 100644 index 000000000000..7518f2b7d47e --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=gpt-345m_seqlen1024_pretrain +dp_degree=1 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-PP4-VPP2-DP2-mbs2-acc8 +device_num=N2C16 +max_iter=50000 +sharding=False +sharding_degree=1 + +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=True + +model=gpt +micro_bs=2 +seed=1234 +acc=8 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh index c5b31056a667..f986f36f4b49 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh @@ -32,7 +32,7 @@ sequence_parallel=True model=gpt micro_bs=8 acc=2 -seed=3589 +seed=1234 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh new file mode 100644 index 000000000000..2c5825c86180 --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=gpt-345m_seqlen1024_pretrain +dp_degree=1 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-PP4-VPP2-DP2-mbs8-acc2 +device_num=N2C16 +max_iter=100 +sharding=False +sharding_degree=1 + +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=False + +model=gpt +micro_bs=8 +acc=2 +seed=1234 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh new file mode 100644 index 000000000000..e20b495fb3b2 --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=gpt-345m_seqlen1024_pretrain +dp_degree=1 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-PP4-VPP2-DP2-mbs8-acc2 +device_num=N2C16 +max_iter=100 +sharding=False +sharding_degree=1 + +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=True + +model=gpt +micro_bs=8 +acc=2 +seed=1234 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh index c1e68d45b7d4..5bebb20853fc 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh @@ -33,8 +33,9 @@ sequence_parallel=True model=gpt micro_bs=8 acc=2 +seed=1234 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} 2>&1; \ No newline at end of file +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file From 049ade1aa346a593089cf73ea217337f9284c3d4 Mon Sep 17 00:00:00 2001 From: iosmers Date: Wed, 10 Jan 2024 14:58:37 +0800 Subject: [PATCH 03/17] add test --- ...seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh | 3 ++- ...seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh | 4 +++- ...qlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh} | 0 ...1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh} | 0 4 files changed, 5 insertions(+), 2 deletions(-) rename tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/{CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh => CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh} (100%) rename tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/{CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh => CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh} (100%) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh index a19e0034f87a..34bdd2527855 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh @@ -31,9 +31,10 @@ sequence_parallel=True model=gpt micro_bs=8 +acc=2 seed=1234 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${seed} 2>&1; \ No newline at end of file +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh index 863f4f2d9f12..ae973e4ea9c6 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh @@ -32,8 +32,10 @@ sequence_parallel=True model=gpt micro_bs=8 +acc=2 +seed=1234 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} 2>&1; \ No newline at end of file +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh similarity index 100% rename from tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh rename to tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh similarity index 100% rename from tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh rename to tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh From d61b877da5cf3c0ae0d77734a11b63d5d5cd79d7 Mon Sep 17 00:00:00 2001 From: iosmers Date: Wed, 10 Jan 2024 15:05:41 +0800 Subject: [PATCH 04/17] add test --- ...024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh | 6 +++--- ...en1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh} | 0 ...24_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/{gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh => gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh} (100%) rename tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/{gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh => gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh} (100%) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh index 7518f2b7d47e..359b04f2d0e1 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh @@ -18,7 +18,7 @@ mp_degree=2 pp_degree=4 bs_item=32 fp_item=bf16 -run_mode=MP2-PP4-VPP2-DP2-mbs2-acc8 +run_mode=MP2-SP2-PP4-VPP2-DP2-mbs8-acc2 device_num=N2C16 max_iter=50000 sharding=False @@ -31,9 +31,9 @@ use_pipeline_parallel=True sequence_parallel=True model=gpt -micro_bs=2 +micro_bs=8 +acc=2 seed=1234 -acc=8 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh similarity index 100% rename from tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh rename to tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh similarity index 100% rename from tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh rename to tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh From c95ed71ddabf04aaf5b23c4807eaf9f8fb899c7c Mon Sep 17 00:00:00 2001 From: iosmers Date: Wed, 10 Jan 2024 15:12:19 +0800 Subject: [PATCH 05/17] add test --- ..._bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh | 40 +++++++++++++++++++ ..._MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh} | 0 2 files changed, 40 insertions(+) create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh rename tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/{CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh => CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh} (100%) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh new file mode 100644 index 000000000000..89903244e834 --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh @@ -0,0 +1,40 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=CE_gpt-345m_seqlen1024_pretrain +dp_degree=1 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-PP4-VPP2-SD2-stage1-mbs8-acc2 +device_num=N2C16 +max_iter=50000 +sharding=stage1 +sharding_degree=2 +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=False + +model=gpt +micro_bs=8 +acc=2 +seed=1234 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh similarity index 100% rename from tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh rename to tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh From f1beda33413bfa7c3697e78292ea172edad2cac5 Mon Sep 17 00:00:00 2001 From: iosmers Date: Wed, 10 Jan 2024 15:17:00 +0800 Subject: [PATCH 06/17] add test --- ...024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/{CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh => gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh} (98%) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh similarity index 98% rename from tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh rename to tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh index 89903244e834..e78a05fef402 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh @@ -20,7 +20,7 @@ bs_item=32 fp_item=bf16 run_mode=MP2-PP4-VPP2-SD2-stage1-mbs8-acc2 device_num=N2C16 -max_iter=50000 +max_iter=100 sharding=stage1 sharding_degree=2 virtual_pp_degree=2 From a785e2db70bfbe87df66dfa5aa04e89112a24b95 Mon Sep 17 00:00:00 2001 From: iosmers Date: Wed, 17 Jan 2024 16:54:47 +0800 Subject: [PATCH 07/17] add test --- ..._bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh | 0 ..._bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh | 0 ..._bs32_bf16_MP2-PP2-SD2-Stage1-mbs8-acc2.sh | 2 +- ...in_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh | 2 +- ..._bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh | 40 +++++++++++++++++++ ...s32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh | 4 +- ...in_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh | 2 +- ..._bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh | 2 +- ...s32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh | 2 +- ..._MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh} | 0 10 files changed, 47 insertions(+), 7 deletions(-) rename tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/{N1C8 => N1C4}/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh (100%) rename tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/{N1C8 => N1C4}/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh (100%) create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh rename tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/{gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh => gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh} (100%) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh similarity index 100% rename from tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh rename to tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh similarity index 100% rename from tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh rename to tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP2-SD2-Stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP2-SD2-Stage1-mbs8-acc2.sh index 781e4b9b47c0..25a9fbbd4fb4 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP2-SD2-Stage1-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP2-SD2-Stage1-mbs8-acc2.sh @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -model_item=gpt-345m_seqlen1024_pretrain +model_item=CE_gpt-345m_seqlen1024_pretrain dp_degree=1 mp_degree=2 pp_degree=2 diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh index bd0308d493fb..945e25f72589 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh @@ -13,7 +13,7 @@ # limitations under the License. model_item=CE_gpt-345m_seqlen1024_pretrain -dp_degree=1 +dp_degree=2 mp_degree=2 pp_degree=4 bs_item=32 diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh new file mode 100644 index 000000000000..e78a05fef402 --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh @@ -0,0 +1,40 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=CE_gpt-345m_seqlen1024_pretrain +dp_degree=1 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-PP4-VPP2-SD2-stage1-mbs8-acc2 +device_num=N2C16 +max_iter=100 +sharding=stage1 +sharding_degree=2 +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=False + +model=gpt +micro_bs=8 +acc=2 +seed=1234 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh index 359b04f2d0e1..0780a22428cf 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -model_item=gpt-345m_seqlen1024_pretrain -dp_degree=1 +model_item=CE_gpt-345m_seqlen1024_pretrain +dp_degree=2 mp_degree=2 pp_degree=4 bs_item=32 diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh index 2c5825c86180..a11f014647ca 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh @@ -13,7 +13,7 @@ # limitations under the License. model_item=gpt-345m_seqlen1024_pretrain -dp_degree=1 +dp_degree=2 mp_degree=2 pp_degree=4 bs_item=32 diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh index e78a05fef402..07c6db66409a 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -model_item=CE_gpt-345m_seqlen1024_pretrain +model_item=gpt-345m_seqlen1024_pretrain dp_degree=1 mp_degree=2 pp_degree=4 diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh index e20b495fb3b2..9c2cc1881341 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh @@ -13,7 +13,7 @@ # limitations under the License. model_item=gpt-345m_seqlen1024_pretrain -dp_degree=1 +dp_degree=2 mp_degree=2 pp_degree=4 bs_item=32 diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh similarity index 100% rename from tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh rename to tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh From 8189673207ba60eb6f9d48c3110c79e53256d08f Mon Sep 17 00:00:00 2001 From: iosmers Date: Wed, 17 Jan 2024 19:04:31 +0800 Subject: [PATCH 08/17] add --- ...eqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh | 2 +- ...eqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh | 2 +- .../hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh index fbfb37e900e5..49867da5f665 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh @@ -19,7 +19,7 @@ pp_degree=1 bs_item=32 fp_item=bf16 run_mode=MP2-SP2-SD2-Stage1-mbs8-acc2 -device_num=N1C8 +device_num=N1C4 max_iter=50000 sharding=stage1 sharding_degree=2 diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh index 934e99b7c67e..c03fee9aa097 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C4/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-SD2-Stage1-mbs8-acc2.sh @@ -19,7 +19,7 @@ pp_degree=1 bs_item=32 fp_item=bf16 run_mode=MP2-SP2-SD2-Stage1-mbs8-acc2 -device_num=N1C8 +device_num=N1C4 max_iter=100 sharding=stage1 sharding_degree=2 diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh index 2206efe57af4..8deed0dd365c 100755 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh @@ -25,7 +25,7 @@ function _set_params(){ micro_batch_size=${6:-"2"} # (必选) micro_batch_size global_batch_size=${7:-"16"} # （必选）global_batch_size run_mode=${8:-"DP"} # (必选) MP模型并行|DP数据并行|PP流水线并行|混合并行DP1-MP1-PP1|DP2-MP8-PP2|DP1-MP8-PP4|DP4-MP8-PP1 - device_num=${9:-"N1C1"} # (必选) 使用的卡数量，N1C1|N1C8|N4C32 （4机32卡） + device_num=${9:-"N1C1"} # (必选) 使用的卡数量，N1C1|N1C8|N1C4|N4C32 （4机32卡） profiling=${PROFILING:-"false"} # (必选) Profiling 开关，默认关闭，通过全局变量传递 model_repo="PaddleNLP" # (必选) 模型套件的名字 speed_unit="tokens/s" # (必选)速度指标单位 From b33b3e6ef157d8670e64c1acd5f57d7e5076ebb7 Mon Sep 17 00:00:00 2001 From: iosmers Date: Thu, 18 Jan 2024 22:33:58 +0800 Subject: [PATCH 09/17] add --- ...eqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh | 4 +--- ...eqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh | 4 +--- ...4_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh | 3 ++- ...1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh | 2 +- .../ce_gpt/benchmark_common/run_benchmark.sh | 5 ++++- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh index 34bdd2527855..6803f8b4416a 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh @@ -31,10 +31,8 @@ sequence_parallel=True model=gpt micro_bs=8 -acc=2 -seed=1234 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh index ae973e4ea9c6..863f4f2d9f12 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh @@ -32,10 +32,8 @@ sequence_parallel=True model=gpt micro_bs=8 -acc=2 -seed=1234 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh index e78a05fef402..51dba71e13dd 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh @@ -20,7 +20,8 @@ bs_item=32 fp_item=bf16 run_mode=MP2-PP4-VPP2-SD2-stage1-mbs8-acc2 device_num=N2C16 -max_iter=100 +max_iter=50000 +max_iter=50000 sharding=stage1 sharding_degree=2 virtual_pp_degree=2 diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh index 9c2cc1881341..39cf90051af2 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh @@ -18,7 +18,7 @@ mp_degree=2 pp_degree=4 bs_item=32 fp_item=bf16 -run_mode=MP2-PP4-VPP2-DP2-mbs8-acc2 +run_mode=MP2-SP2-PP4-VPP2-DP2-mbs8-acc2 device_num=N2C16 max_iter=100 sharding=False diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh index 8deed0dd365c..a26284581a24 100755 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh @@ -153,7 +153,10 @@ function _train(){ run_pretrain.py ${train_cmd}" workerlog_id=0 ;; - DP8-mbs2-acc2|SD8-stage1-mbs2-acc2|SD8-stage2-mbs2-acc2|SD8-stage3-mbs2-acc2|PP8-mbs16-acc2|MP2-SD4-stage1-mbs4-acc2|MP2-SP2-PP2-DP2-mbs8-acc2|MP8-mbs16-acc2|MP2-PP2-DP2-mbs8-acc2|MP2-PP2-SD2-Stage1-mbs8-acc2|MP2-SP2-PP2-SD2-Stage1-mbs8-acc2|MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8) echo "run run_mode: ${run_mode}" + DP8-mbs2-acc2|SD8-stage1-mbs2-acc2|SD8-stage2-mbs2-acc2|SD8-stage3-mbs2-acc2|PP8-mbs16-acc2|MP8-mbs16-acc2| + MP2-PP2-DP2-mbs8-acc2|MP2-PP2-SD2-Stage1-mbs8-acc2|MP2-SD4-stage1-mbs4-acc2|MP2-SP2-PP2-DP2-mbs8-acc2|MP2-SP2-PP2-SD2-Stage1-mbs8-acc2| + MP2-PP4-VPP2-DP2-mbs8-acc2|MP2-PP4-VPP2-SD2-stage1-mbs8-acc2|MP2-SP2-PP4-VPP2-DP2-mbs8-acc2|MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2) + echo "run run_mode: ${run_mode}" train_cmd="python -m paddle.distributed.launch --log_dir=./mylog --devices=0,1,2,3,4,5,6,7 ${PADDLE_RANK_OPTION}\ run_pretrain.py ${train_cmd}" workerlog_id=0 From ecdbd0fdda8c015de2525cc2eec1e9b468c154aa Mon Sep 17 00:00:00 2001 From: iosmers Date: Thu, 18 Jan 2024 22:39:02 +0800 Subject: [PATCH 10/17] add --- .../hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh index a26284581a24..bda6194a3358 100755 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh @@ -154,6 +154,7 @@ function _train(){ workerlog_id=0 ;; DP8-mbs2-acc2|SD8-stage1-mbs2-acc2|SD8-stage2-mbs2-acc2|SD8-stage3-mbs2-acc2|PP8-mbs16-acc2|MP8-mbs16-acc2| + MP2-PP2-DP2-mbs8-acc2|MP2-PP2-SD2-Stage1-mbs8-acc2|MP2-SD4-stage1-mbs4-acc2|MP2-SP2-PP2-DP2-mbs8-acc2|MP2-SP2-PP2-SD2-Stage1-mbs8-acc2| MP2-PP4-VPP2-DP2-mbs8-acc2|MP2-PP4-VPP2-SD2-stage1-mbs8-acc2|MP2-SP2-PP4-VPP2-DP2-mbs8-acc2|MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2) echo "run run_mode: ${run_mode}" From 07238655db56753df2cf29f6e1223a604fca7e34 Mon Sep 17 00:00:00 2001 From: iosmers Date: Thu, 18 Jan 2024 22:44:04 +0800 Subject: [PATCH 11/17] add --- .../hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh index bda6194a3358..a26284581a24 100755 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh @@ -154,7 +154,6 @@ function _train(){ workerlog_id=0 ;; DP8-mbs2-acc2|SD8-stage1-mbs2-acc2|SD8-stage2-mbs2-acc2|SD8-stage3-mbs2-acc2|PP8-mbs16-acc2|MP8-mbs16-acc2| - MP2-PP2-DP2-mbs8-acc2|MP2-PP2-SD2-Stage1-mbs8-acc2|MP2-SD4-stage1-mbs4-acc2|MP2-SP2-PP2-DP2-mbs8-acc2|MP2-SP2-PP2-SD2-Stage1-mbs8-acc2| MP2-PP4-VPP2-DP2-mbs8-acc2|MP2-PP4-VPP2-SD2-stage1-mbs8-acc2|MP2-SP2-PP4-VPP2-DP2-mbs8-acc2|MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2) echo "run run_mode: ${run_mode}" From 6d667214a8068cd956faad1f5f6c094ff1482842 Mon Sep 17 00:00:00 2001 From: iosmers Date: Thu, 18 Jan 2024 22:45:37 +0800 Subject: [PATCH 12/17] add --- .../ce_gpt/benchmark_common/run_benchmark.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh index a26284581a24..71d93c3c8062 100755 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh @@ -153,9 +153,7 @@ function _train(){ run_pretrain.py ${train_cmd}" workerlog_id=0 ;; - DP8-mbs2-acc2|SD8-stage1-mbs2-acc2|SD8-stage2-mbs2-acc2|SD8-stage3-mbs2-acc2|PP8-mbs16-acc2|MP8-mbs16-acc2| - MP2-PP2-DP2-mbs8-acc2|MP2-PP2-SD2-Stage1-mbs8-acc2|MP2-SD4-stage1-mbs4-acc2|MP2-SP2-PP2-DP2-mbs8-acc2|MP2-SP2-PP2-SD2-Stage1-mbs8-acc2| - MP2-PP4-VPP2-DP2-mbs8-acc2|MP2-PP4-VPP2-SD2-stage1-mbs8-acc2|MP2-SP2-PP4-VPP2-DP2-mbs8-acc2|MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2) + DP8-mbs2-acc2|SD8-stage1-mbs2-acc2|SD8-stage2-mbs2-acc2|SD8-stage3-mbs2-acc2|PP8-mbs16-acc2|MP8-mbs16-acc2|MP2-PP2-DP2-mbs8-acc2|MP2-PP2-SD2-Stage1-mbs8-acc2|MP2-SD4-stage1-mbs4-acc2|MP2-SP2-PP2-DP2-mbs8-acc2|MP2-SP2-PP2-SD2-Stage1-mbs8-acc2|MP2-PP4-VPP2-DP2-mbs8-acc2|MP2-PP4-VPP2-SD2-stage1-mbs8-acc2|MP2-SP2-PP4-VPP2-DP2-mbs8-acc2|MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2) echo "run run_mode: ${run_mode}" train_cmd="python -m paddle.distributed.launch --log_dir=./mylog --devices=0,1,2,3,4,5,6,7 ${PADDLE_RANK_OPTION}\ run_pretrain.py ${train_cmd}" From 29a3a0235538c910ca9224bab27c4bb8f63fee2c Mon Sep 17 00:00:00 2001 From: iosmers Date: Thu, 18 Jan 2024 22:49:36 +0800 Subject: [PATCH 13/17] add --- ...pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh index 5bebb20853fc..c1e68d45b7d4 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh @@ -33,9 +33,8 @@ sequence_parallel=True model=gpt micro_bs=8 acc=2 -seed=1234 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} 2>&1; \ No newline at end of file From bb256063d5dbd21d85aa01c0b9c2fbfe637bdc18 Mon Sep 17 00:00:00 2001 From: iosmers Date: Thu, 18 Jan 2024 22:52:22 +0800 Subject: [PATCH 14/17] add --- ...4_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh index f986f36f4b49..7578f5868efd 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh @@ -32,7 +32,6 @@ sequence_parallel=True model=gpt micro_bs=8 acc=2 -seed=1234 bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh # run From 5762fe67e703e3a7345348c445c6c31de8e045c6 Mon Sep 17 00:00:00 2001 From: iosmers Date: Mon, 22 Jan 2024 13:38:24 +0800 Subject: [PATCH 15/17] update --- ...in_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh | 41 +++++++++++++++++++ ..._bf16_MP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh | 41 +++++++++++++++++++ ...s32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh | 41 +++++++++++++++++++ ...6_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh | 39 ++++++++++++++++++ ...in_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh | 41 +++++++++++++++++++ ..._bf16_MP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh | 40 ++++++++++++++++++ ...s32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh | 41 +++++++++++++++++++ ...6_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh | 40 ++++++++++++++++++ 8 files changed, 324 insertions(+) create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh create mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh new file mode 100644 index 000000000000..fd26253ed46f --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=CE_gpt-345m_seqlen1024_pretrain +dp_degree=2 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-PP4-VPP2-DP2-mbs2-acc8 +device_num=N2C16 +max_iter=50000 +sharding=False +sharding_degree=1 + +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=False + +model=gpt +micro_bs=2 +acc=8 +seed=1234 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh new file mode 100644 index 000000000000..226a0ab4cbce --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=CE_gpt-345m_seqlen1024_pretrain +dp_degree=1 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-PP4-VPP2-SD2-stage1-mbs2-acc8 +device_num=N2C16 +max_iter=50000 +max_iter=50000 +sharding=stage1 +sharding_degree=2 +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=False + +model=gpt +micro_bs=2 +acc=8 +seed=1234 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh new file mode 100644 index 000000000000..38a54e048d7c --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=CE_gpt-345m_seqlen1024_pretrain +dp_degree=2 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-SP2-PP4-VPP2-DP2-mbs2-acc8 +device_num=N2C16 +max_iter=50000 +sharding=False +sharding_degree=1 + +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=True + +model=gpt +micro_bs=2 +acc=8 +seed=1234 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh new file mode 100644 index 000000000000..cfc41c6eaedc --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh @@ -0,0 +1,39 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=CE_gpt-345m_seqlen1024_pretrain +dp_degree=1 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8 +device_num=N2C16 +max_iter=50000 +sharding=stage1 +sharding_degree=2 +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=True + +model=gpt +micro_bs=2 +acc=8 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh new file mode 100644 index 000000000000..d3573c0c4994 --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs2-acc8.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=gpt-345m_seqlen1024_pretrain +dp_degree=2 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-PP4-VPP2-DP2-mbs2-acc8 +device_num=N2C16 +max_iter=100 +sharding=False +sharding_degree=1 + +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=False + +model=gpt +micro_bs=2 +acc=8 +seed=1234 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh new file mode 100644 index 000000000000..7e4e84cd4e62 --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh @@ -0,0 +1,40 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=gpt-345m_seqlen1024_pretrain +dp_degree=1 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-PP4-VPP2-SD2-stage1-mbs2-acc8 +device_num=N2C16 +max_iter=100 +sharding=stage1 +sharding_degree=2 +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=False + +model=gpt +micro_bs=2 +acc=8 +seed=1234 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh new file mode 100644 index 000000000000..cf187c0fd561 --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs2-acc8.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=gpt-345m_seqlen1024_pretrain +dp_degree=2 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-SP2-PP4-VPP2-DP2-mbs2-acc8 +device_num=N2C16 +max_iter=100 +sharding=False +sharding_degree=1 + +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=True + +model=gpt +micro_bs=2 +acc=8 +seed=1234 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh new file mode 100644 index 000000000000..03a50a587e9e --- /dev/null +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8.sh @@ -0,0 +1,40 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +model_item=gpt-345m_seqlen1024_pretrain +dp_degree=1 +mp_degree=2 +pp_degree=4 +bs_item=32 +fp_item=bf16 +run_mode=MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8 +device_num=N2C16 +max_iter=100 +sharding=stage1 +sharding_degree=2 + +virtual_pp_degree=2 +use_recompute=True +eval_freq=25 +use_pipeline_parallel=True +sequence_parallel=True + +model=gpt +micro_bs=2 +acc=8 + +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh +# run +bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ +${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} 2>&1; \ No newline at end of file From a3a29459db7529fdc8e0565e401feb6926819a9b Mon Sep 17 00:00:00 2001 From: iosmers Date: Mon, 22 Jan 2024 13:43:41 +0800 Subject: [PATCH 16/17] add --- ...in_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh | 41 ------------------- ..._bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh | 41 ------------------- ...s32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh | 41 ------------------- ...6_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh | 39 ------------------ ...in_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh | 41 ------------------- ..._bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh | 40 ------------------ ...s32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh | 41 ------------------- ...6_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh | 40 ------------------ 8 files changed, 324 deletions(-) delete mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh delete mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh delete mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh delete mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh delete mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh delete mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh delete mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh delete mode 100644 tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh deleted file mode 100644 index 945e25f72589..000000000000 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model_item=CE_gpt-345m_seqlen1024_pretrain -dp_degree=2 -mp_degree=2 -pp_degree=4 -bs_item=32 -fp_item=bf16 -run_mode=MP2-PP4-VPP2-DP2-mbs8-acc2 -device_num=N2C16 -max_iter=50000 -sharding=False -sharding_degree=1 - -virtual_pp_degree=2 -use_recompute=True -eval_freq=25 -use_pipeline_parallel=True -sequence_parallel=False - -model=gpt -micro_bs=8 -acc=2 -seed=1234 - -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh -# run -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh deleted file mode 100644 index 51dba71e13dd..000000000000 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model_item=CE_gpt-345m_seqlen1024_pretrain -dp_degree=1 -mp_degree=2 -pp_degree=4 -bs_item=32 -fp_item=bf16 -run_mode=MP2-PP4-VPP2-SD2-stage1-mbs8-acc2 -device_num=N2C16 -max_iter=50000 -max_iter=50000 -sharding=stage1 -sharding_degree=2 -virtual_pp_degree=2 -use_recompute=True -eval_freq=25 -use_pipeline_parallel=True -sequence_parallel=False - -model=gpt -micro_bs=8 -acc=2 -seed=1234 - -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh -# run -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh deleted file mode 100644 index 0780a22428cf..000000000000 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model_item=CE_gpt-345m_seqlen1024_pretrain -dp_degree=2 -mp_degree=2 -pp_degree=4 -bs_item=32 -fp_item=bf16 -run_mode=MP2-SP2-PP4-VPP2-DP2-mbs8-acc2 -device_num=N2C16 -max_iter=50000 -sharding=False -sharding_degree=1 - -virtual_pp_degree=2 -use_recompute=True -eval_freq=25 -use_pipeline_parallel=True -sequence_parallel=True - -model=gpt -micro_bs=8 -acc=2 -seed=1234 - -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh -# run -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh deleted file mode 100644 index 7578f5868efd..000000000000 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model_item=CE_gpt-345m_seqlen1024_pretrain -dp_degree=1 -mp_degree=2 -pp_degree=4 -bs_item=32 -fp_item=bf16 -run_mode=MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2 -device_num=N2C16 -max_iter=50000 -sharding=stage1 -sharding_degree=2 -virtual_pp_degree=2 -use_recompute=True -eval_freq=25 -use_pipeline_parallel=True -sequence_parallel=True - -model=gpt -micro_bs=8 -acc=2 - -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh -# run -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh deleted file mode 100644 index a11f014647ca..000000000000 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-DP2-mbs8-acc2.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model_item=gpt-345m_seqlen1024_pretrain -dp_degree=2 -mp_degree=2 -pp_degree=4 -bs_item=32 -fp_item=bf16 -run_mode=MP2-PP4-VPP2-DP2-mbs8-acc2 -device_num=N2C16 -max_iter=100 -sharding=False -sharding_degree=1 - -virtual_pp_degree=2 -use_recompute=True -eval_freq=25 -use_pipeline_parallel=True -sequence_parallel=False - -model=gpt -micro_bs=8 -acc=2 -seed=1234 - -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh -# run -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh deleted file mode 100644 index 07c6db66409a..000000000000 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model_item=gpt-345m_seqlen1024_pretrain -dp_degree=1 -mp_degree=2 -pp_degree=4 -bs_item=32 -fp_item=bf16 -run_mode=MP2-PP4-VPP2-SD2-stage1-mbs8-acc2 -device_num=N2C16 -max_iter=100 -sharding=stage1 -sharding_degree=2 -virtual_pp_degree=2 -use_recompute=True -eval_freq=25 -use_pipeline_parallel=True -sequence_parallel=False - -model=gpt -micro_bs=8 -acc=2 -seed=1234 - -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh -# run -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh deleted file mode 100644 index 39cf90051af2..000000000000 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-DP2-mbs8-acc2.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model_item=gpt-345m_seqlen1024_pretrain -dp_degree=2 -mp_degree=2 -pp_degree=4 -bs_item=32 -fp_item=bf16 -run_mode=MP2-SP2-PP4-VPP2-DP2-mbs8-acc2 -device_num=N2C16 -max_iter=100 -sharding=False -sharding_degree=1 - -virtual_pp_degree=2 -use_recompute=True -eval_freq=25 -use_pipeline_parallel=True -sequence_parallel=True - -model=gpt -micro_bs=8 -acc=2 -seed=1234 - -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh -# run -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} ${seed} 2>&1; \ No newline at end of file diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh deleted file mode 100644 index c1e68d45b7d4..000000000000 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N2C16/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2.sh +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model_item=gpt-345m_seqlen1024_pretrain -dp_degree=1 -mp_degree=2 -pp_degree=4 -bs_item=32 -fp_item=bf16 -run_mode=MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2 -device_num=N2C16 -max_iter=100 -sharding=stage1 -sharding_degree=2 - -virtual_pp_degree=2 -use_recompute=True -eval_freq=25 -use_pipeline_parallel=True -sequence_parallel=True - -model=gpt -micro_bs=8 -acc=2 - -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh -# run -bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \ -${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} ${acc} 2>&1; \ No newline at end of file From a798a6a52f4d5794ed977fedd75ad3bfc6b51281 Mon Sep 17 00:00:00 2001 From: iosmers Date: Mon, 22 Jan 2024 14:06:42 +0800 Subject: [PATCH 17/17] add --- .../hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh index 71d93c3c8062..11ef0dce5822 100755 --- a/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh +++ b/tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh @@ -153,7 +153,7 @@ function _train(){ run_pretrain.py ${train_cmd}" workerlog_id=0 ;; - DP8-mbs2-acc2|SD8-stage1-mbs2-acc2|SD8-stage2-mbs2-acc2|SD8-stage3-mbs2-acc2|PP8-mbs16-acc2|MP8-mbs16-acc2|MP2-PP2-DP2-mbs8-acc2|MP2-PP2-SD2-Stage1-mbs8-acc2|MP2-SD4-stage1-mbs4-acc2|MP2-SP2-PP2-DP2-mbs8-acc2|MP2-SP2-PP2-SD2-Stage1-mbs8-acc2|MP2-PP4-VPP2-DP2-mbs8-acc2|MP2-PP4-VPP2-SD2-stage1-mbs8-acc2|MP2-SP2-PP4-VPP2-DP2-mbs8-acc2|MP2-SP2-PP4-VPP2-SD2-stage1-mbs8-acc2) + DP8-mbs2-acc2|SD8-stage1-mbs2-acc2|SD8-stage2-mbs2-acc2|SD8-stage3-mbs2-acc2|PP8-mbs16-acc2|MP8-mbs16-acc2|MP2-PP2-DP2-mbs8-acc2|MP2-PP2-SD2-Stage1-mbs8-acc2|MP2-SD4-stage1-mbs4-acc2|MP2-SP2-PP2-DP2-mbs8-acc2|MP2-SP2-PP2-SD2-Stage1-mbs8-acc2|MP2-PP4-VPP2-DP2-mbs2-acc8|MP2-PP4-VPP2-SD2-stage1-mbs2-acc8|MP2-SP2-PP4-VPP2-DP2-mbs2-acc8|MP2-SP2-PP4-VPP2-SD2-stage1-mbs2-acc8) echo "run run_mode: ${run_mode}" train_cmd="python -m paddle.distributed.launch --log_dir=./mylog --devices=0,1,2,3,4,5,6,7 ${PADDLE_RANK_OPTION}\ run_pretrain.py ${train_cmd}"