From d98e9e7b864e6024aaae429f9690d782d69c03db Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Wed, 19 Jun 2024 09:33:42 +0800 Subject: [PATCH] Remove delay_scale_loss and release_grads for llama-2 13B's benchmark. --- .../pretrain-llama2_13b-auto_tuner.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_13b/pretrain-llama2_13b-auto_tuner.json b/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_13b/pretrain-llama2_13b-auto_tuner.json index 011e0a8c44e1..0a39212b492f 100644 --- a/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_13b/pretrain-llama2_13b-auto_tuner.json +++ b/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_13b/pretrain-llama2_13b-auto_tuner.json @@ -10,8 +10,8 @@ "pipeline_parallel_degree": 4, "sharding": "stage1", "sharding_parallel_config": "split_param enable_stage1_overlap enable_stage1_allgather_overlap", - "tensor_parallel_config": "enable_delay_scale_loss enable_mp_async_allreduce enable_mp_skip_c_identity enable_mp_fused_linear_param_grad_add", - "pipeline_parallel_config": "enable_delay_scale_loss enable_sharding_comm_overlap enable_release_grads", + "tensor_parallel_config": "enable_mp_async_allreduce enable_mp_skip_c_identity enable_mp_fused_linear_param_grad_add", + "pipeline_parallel_config": "enable_sharding_comm_overlap", "virtual_pp_degree": 5, "sequence_parallel": 0, "use_flash_attention": true,