diff --git a/paddlenlp/transformers/llama/modeling_auto.py b/paddlenlp/transformers/llama/modeling_auto.py
index 64277d744016..fce60999c214 100644
--- a/paddlenlp/transformers/llama/modeling_auto.py
+++ b/paddlenlp/transformers/llama/modeling_auto.py
@@ -181,6 +181,7 @@ def scaled_dot_product_attention(
 
         attn_output = paddle.matmul(attn_weights, value_states)
         attn_output = attn_output.transpose([0, 2, 1, 3])
+        # [bsz, q_len, num_heads, head_dim] -> [bsz, q_len, num_heads * head_dim]
         attn_output = attn_output.reshape([bsz, q_len, head_dim * num_heads])
         return (attn_output, attn_weights) if output_attentions else attn_output
 
@@ -399,9 +400,10 @@ def forward(
         alibi: Optional[paddle.Tensor] = None,
     ) -> Tuple[paddle.Tensor, Optional[paddle.Tensor], Optional[Tuple[paddle.Tensor]]]:
         """Input shape: Batch x Time x Channel"""
-        # [bs, seq_len, num_head * head_dim] -> [seq_len / n, bs, num_head * head_dim] (n is model parallelism)
+        # [bs, seq_len, num_head * head_dim] or [seq_len / n, bs, num_head * head_dim] (if sequence_parallel)
         # enter tp region
         if self.config.sequence_parallel:
+            # [seq_len / n, bs, num_head * head_dim] -> [seq_len, bs, num_head * head_dim] (if sequence_parallel)
             hidden_states = dist.reshard(
                 hidden_states,
                 get_mesh(self.ipp),
@@ -422,6 +424,8 @@ def forward(
             value_states = self.v_proj(hidden_states).reshape(shape=target_key_value_shape)
 
         if self.config.sequence_parallel:
+            # [seq_len, bs, num_head * head_dim] -> [bs, seq_len, num_head * head_dim]  (if sequence_parallel)
+            # FA and rope not support sequence first
             query_states = paddle.transpose(query_states, [1, 0, 2, 3])
             key_states = paddle.transpose(key_states, [1, 0, 2, 3])
             value_states = paddle.transpose(value_states, [1, 0, 2, 3])
@@ -526,12 +530,12 @@ def forward(
         else:
             attn_output = outputs
 
-        # if sequence_parallel is true, out shape are [q_len / n, bs, num_head * head_dim]
-        # else their shape are [bs, q_len, num_head * head_dim], n is mp parallelism.
+        # [bs, q_len, num_head * head_dim]
         attn_output = self.o_proj(attn_output)
 
         # enter sp region
         if self.config.sequence_parallel:
+            # [bs, q_len, num_head * head_dim] -> [q_len / n, bs, num_head * head_dim]
             attn_output = paddle.transpose(attn_output, [1, 0, 2])
             attn_output = dist.reshard(
                 attn_output,
@@ -595,7 +599,7 @@ def forward(
             cache (`Tuple(paddle.Tensor)`, *optional*): cached past key and value projection states
         """
 
-        # [bs * seq_len, embed_dim] -> [seq_len * bs / n, embed_dim] (sequence_parallel)
+        # [bs, seq_len, embed_dim] or [seq_len / n, bs, embed_dim] (if sequence_parallel)
         residual = hidden_states
 
         hidden_states = self.input_layernorm(hidden_states)
diff --git a/requirements.txt b/requirements.txt
index 4b676d900563..33b7f4bf9149 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -23,3 +23,4 @@ safetensors
 tool_helpers
 aistudio-sdk>=0.1.3
 jinja2
+regex
\ No newline at end of file
diff --git a/scripts/distribute/ci_case_auto.sh b/scripts/distribute/ci_case_auto.sh
index 8707289b7fe2..7dde1396d725 100755
--- a/scripts/distribute/ci_case_auto.sh
+++ b/scripts/distribute/ci_case_auto.sh
@@ -28,6 +28,16 @@ export llm_gpt_case_path=$root_path/llm/gpt-3/auto_parallel
 
 unset CUDA_VISIBLE_DEVICES
 
+function is_a100() {
+    if [ $(nvidia-smi|grep A100|wc -l)  -ne 0 ];then
+        echo 1
+    else
+        echo 0
+    fi
+}
+
+IS_A100=$(is_a100)
+
 function gpt_case_list_auto() {
     gpt_auto_recompute_bs16_fp32_DP1-MP1-PP1
     gpt_auto_recompute_bs16_fp16_o2_DP1-MP1-PP8
@@ -108,6 +118,11 @@ function gpt_auto_recompute_bs16_fp32_DP1-MP1-PP1() {
     loss_base=10.507633305
     ips_base=3518
     mem_base=11750.6
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.530449009
+        ips_base=16763
+        mem_base=11750.6
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -144,6 +159,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP1-MP1-PP8() {
     loss_base=10.570028400
     ips_base=35050
     mem_base=1988.9
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.559662151
+        ips_base=83918
+        mem_base=2022.7
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -181,6 +201,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP1-MP1-PP8_pir() {
     loss_base=10.570028400
     ips_base=35050
     mem_base=1988.9
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.559662151
+        ips_base=83918
+        mem_base=2022.7
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -217,6 +242,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP1-MP2-PP4() {
     loss_base=10.700293922
     ips_base=32518
     mem_base=1535.7
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.679453373
+        ips_base=79116
+        mem_base=1488.2
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -253,6 +283,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP2-MP2-PP2() {
     loss_base=10.672543240
     ips_base=18681
     mem_base=2135.7
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.651049423
+        ips_base=41174
+        mem_base=2064.5
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -290,6 +325,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP2-MP2-PP2_pir() {
     loss_base=10.672543240
     ips_base=18681
     mem_base=2135.7
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.651049423
+        ips_base=41174
+        mem_base=2064.5
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -326,6 +366,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP4-MP2-Sharding4_stage1() {
     loss_base=10.720068359
     ips_base=15232
     mem_base=1999.2
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.657777309
+        ips_base=30027
+        mem_base=2002.0
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -363,6 +408,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP4-MP2-Sharding4_stage1_pir() {
     loss_base=10.720068359
     ips_base=15232
     mem_base=1999.2
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.657777309
+        ips_base=30027
+        mem_base=2002.0
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -399,6 +449,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP4-MP2-Sharding4_stage2() {
     loss_base=10.720078850
     ips_base=15571
     mem_base=1999.2
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.657803535
+        ips_base=29166
+        mem_base=2002.0
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -435,6 +490,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP4-MP2-Sharding4_stage3() {
     loss_base=10.681921577
     ips_base=13813
     mem_base=1747.6
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.662137604
+        ips_base=24700
+        mem_base=1750.5
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -471,6 +531,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP2-MP1-PP4_Sharding2_stage1() {
     loss_base=10.579057693
     ips_base=19822
     mem_base=1709.8
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.586785984
+        ips_base=42813
+        mem_base=1743.8
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -508,6 +573,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP2-MP1-PP4_Sharding2_stage1_pir() {
     loss_base=10.579057693
     ips_base=19822
     mem_base=1709.8
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.586785984
+        ips_base=42813
+        mem_base=1743.8
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -544,6 +614,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP2-MP1-PP4_Sharding2_stage2() {
     loss_base=10.579057693
     ips_base=20170
     mem_base=1709.8
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.586785984
+        ips_base=42995
+        mem_base=1743.8
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -580,6 +655,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP2-MP1-PP4_Sharding2_stage3() {
     loss_base=10.585316849
     ips_base=15742
     mem_base=1591.6
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.555718899
+        ips_base=34688
+        mem_base=1625.6
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -616,6 +696,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP2-MP2-PP2_Sharding2_stage1() {
     loss_base=10.672568035
     ips_base=19461
     mem_base=1384.7
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.651032448
+        ips_base=42435
+        mem_base=1377.5
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -652,6 +737,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP2-MP2-PP2_Sharding2_stage2() {
     loss_base=10.672568035
     ips_base=19652
     mem_base=1384.7
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.651032448
+        ips_base=43008
+        mem_base=1377.5
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -689,6 +779,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP2-MP2-PP2_Sharding2_stage2_pir() {
     loss_base=10.672568035
     ips_base=19652
     mem_base=1384.7
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.651032448
+        ips_base=43008
+        mem_base=1377.5
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -725,6 +820,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP2-MP2-PP2_Sharding2_stage3() {
     loss_base=10.696336079
     ips_base=16613
     mem_base=1280.5
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.705118465
+        ips_base=37104
+        mem_base=1217.3
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -762,6 +862,11 @@ function gpt_auto_recompute_bs16_fp16_o2_DP2-MP2-PP2_Sharding2_stage3_pir() {
     loss_base=10.696336079
     ips_base=16613
     mem_base=1280.5
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.705118465
+        ips_base=37104
+        mem_base=1217.3
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -843,8 +948,12 @@ function gpt_auto_sp_acc_check() {
     loss_base=`cat ${log_dir_spFalse}/workerlog.0 |  grep '30/30' | awk -F 'loss: ' '{print $2}' | awk -F ',' '{print $1}'`
     ips_base=-1
     mem_base=-1
+    allclose=0
     echo "result: loss_spTrue=$loss loss_spFasle=$loss_base"
-    check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
+    if [ $IS_A100 -ne 0 ];then
+        allclose=1
+    fi
+    check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem} ${allclose}
     echo "=========== $FUNCNAME run  end ==========="
 }
 
@@ -908,6 +1017,9 @@ function llama_static_auto_recompute_bs8_fp32_DP1-MP1-PP1() {
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem"
     loss_base=9.52110565
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=9.54202747
+    fi
     ips_base=-1
     mem_base=-1
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
@@ -974,6 +1086,9 @@ function llama_static_auto_recompute_bs16_fp32_DP2-MP1-PP1() {
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem"
     loss_base=9.42011833
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=9.44003963
+    fi
     ips_base=-1
     mem_base=-1
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
@@ -1040,6 +1155,9 @@ function llama_static_auto_recompute_bs16_fp32_DP2-MP2-PP1() {
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem"
     loss_base=9.44299471
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=9.45633757
+    fi
     ips_base=-1
     mem_base=-1
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
@@ -1106,6 +1224,9 @@ function llama_static_auto_recompute_bs16_fp32_DP2-MP2-PP2() {
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem"
     loss_base=9.45936012
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=9.46121407
+    fi
     ips_base=-1
     mem_base=-1
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
@@ -1174,6 +1295,9 @@ function llama_static_auto_recompute_bs16_fp32_DP2-MP2-PP2-VPP2-Sharding2_stage2
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem"
     loss_base=9.46707726
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=9.44474411
+    fi
     ips_base=-1
     mem_base=-1
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
@@ -1243,6 +1367,9 @@ function llama_static_auto_recompute_bs16_fp16_DP2-MP2-PP2-VPP2-Sharding2_stage2
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem"
     loss_base=10.0859375
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.125
+    fi
     ips_base=-1
     mem_base=-1
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
@@ -1310,6 +1437,9 @@ function llama_dygraph_auto_bs8_fp32_DP2() {
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem"
     loss_base=9.53389835
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=9.54253578
+    fi
     ips_base=-1
     mem_base=-1
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
@@ -1377,6 +1507,9 @@ function llama_dygraph_auto_bs8_fp32_DP2-MP2() {
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem"
     loss_base=9.39066124
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=9.41613197
+    fi
     ips_base=-1
     mem_base=-1
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
@@ -1444,6 +1577,9 @@ function llama_dygraph_auto_bs8_fp32_DP2-MP2-PP2() {
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem"
     loss_base=9.38235474
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=9.4053154
+    fi
     ips_base=-1
     mem_base=-1
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
@@ -1512,6 +1648,9 @@ function llama_dygraph_auto_bs8_fp16_DP2-MP2-PP2() {
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem"
     loss_base=9.38256836
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=9.4055109
+    fi
     ips_base=-1
     mem_base=-1
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
@@ -1571,7 +1710,7 @@ function llm_gpt_dygraph_auto_bs8_fp32_DP2() {
         --enable_auto_parallel 1 \
         --to_static 0 \
         --fp16 0 \
-        --fp16_opt_level "O2"
+        --fp16_opt_level "O2" \
         >>${log_path}/$FUNCNAME 2>&1
     loss=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'loss: ' '{print $2}' | awk -F ',' '{print $1}'`
     loss_md5=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'loss_md5: ' '{print $2}' | awk -F ',' '{print $1}'`
@@ -1582,6 +1721,9 @@ function llm_gpt_dygraph_auto_bs8_fp32_DP2() {
     loss_md5_base=0ebf68698887b33b33a46518621cf412
     ips_base=-1
     mem_base=-1
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.58541679
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -1641,7 +1783,7 @@ function llm_gpt_dygraph_auto_bs8_fp32_DP2-MP2() {
         --enable_auto_parallel 1 \
         --to_static 0 \
         --fp16 0 \
-        --fp16_opt_level "O2"
+        --fp16_opt_level "O2" \
         >>${log_path}/$FUNCNAME 2>&1
     loss=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'loss: ' '{print $2}' | awk -F ',' '{print $1}'`
     loss_md5=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'loss_md5: ' '{print $2}' | awk -F ',' '{print $1}'`
@@ -1652,6 +1794,9 @@ function llm_gpt_dygraph_auto_bs8_fp32_DP2-MP2() {
     loss_md5_base=6df87d01bd08113a92930f6349514b35
     ips_base=-1
     mem_base=-1
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.58452606
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -1711,7 +1856,7 @@ function llm_gpt_dygraph_auto_bs8_fp32_DP2-MP2-PP2() {
         --enable_auto_parallel 1 \
         --to_static 0 \
         --fp16 0 \
-        --fp16_opt_level "O2"
+        --fp16_opt_level "O2" \
         >>${log_path}/$FUNCNAME 2>&1
     loss=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'loss: ' '{print $2}' | awk -F ',' '{print $1}'`
     loss_md5=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'loss_md5: ' '{print $2}' | awk -F ',' '{print $1}'`
@@ -1722,6 +1867,9 @@ function llm_gpt_dygraph_auto_bs8_fp32_DP2-MP2-PP2() {
     loss_md5_base=6cb4e151b35f026190df90ab240d9a95
     ips_base=-1
     mem_base=-1
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.57996178
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -1781,7 +1929,7 @@ function llm_gpt_dygraph_auto_bs8_fp16_DP2-MP2-PP2() {
         --enable_auto_parallel 1 \
         --to_static 0 \
         --fp16 1 \
-        --fp16_opt_level "O2"
+        --fp16_opt_level "O2" \
         >>${log_path}/$FUNCNAME 2>&1
     loss=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'loss: ' '{print $2}' | awk -F ',' '{print $1}'`
     loss_md5=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'loss_md5: ' '{print $2}' | awk -F ',' '{print $1}'`
@@ -1792,6 +1940,9 @@ function llm_gpt_dygraph_auto_bs8_fp16_DP2-MP2-PP2() {
     loss_md5_base=e82a1f5668870d18a2d45b3ee0a25386
     ips_base=-1
     mem_base=-1
+    if [ $IS_A100 -ne 0 ];then
+        loss_base=10.58061218
+    fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
 }
@@ -1805,7 +1956,7 @@ function check_result() {
         exit -1
     fi
 
-    if [ $# -ne 7 ]; then
+    if [ $# -ne 7 ] && [ $# -ne 8 ]; then
         echo -e "\033[31m $1 parameter transfer failed: $@ \033[0m" | tee -a ${log_path}/result.log
         exit -1
     fi
@@ -1813,8 +1964,17 @@ function check_result() {
     diff_loss=$(echo $2 $3|awk '{printf "%0.2f\n", ($2-$1)/$1*100}')
     echo -e "loss_base: $2 loss_test: $3 loss_diff: $diff_loss%" | tee -a ${log_path}/result.log
     if [ $2 != $3 ];then
-        echo -e "\033[31m $1 loss diff check failed! \033[0m" | tee -a ${log_path}/result.log
-        exit -1
+        if [ -z "$8" ] || [ $8 -ne 1 ] ;then
+            echo -e "\033[31m $1 loss diff check failed! \033[0m" | tee -a ${log_path}/result.log
+            exit -1
+        else
+            diff=$(echo "$2 $3" | awk '{print $1-$2}')
+            gt=$(echo "${diff#-} 1e-5" | awk '{print ($1>$2)?"1":"0"}')
+            if [ $gt -eq 1 ];then
+                echo -e "\033[31m $1 loss diff check failed! \033[0m" | tee -a ${log_path}/result.log
+                exit -1
+            fi
+        fi
     fi
 
     diff_ips=$(echo $4 $5|awk '{printf "%0.2f\n", ($2-$1)/$1*100}')