Skip to content

Commit 3971fc7

Browse files
waliwali777waliwali777
authored
[AutoParallel] Fix EXCODE bug of AutoParallel CI (#9355)
* Fix exitcode bug * Fix `track_case_status` func match bug * Fix return code * Fix print_info func with exit -6 * set output format of fail tests modify verification check failed
1 parent 0977858 commit 3971fc7

File tree

3 files changed

+38
-26
lines changed

3 files changed

+38
-26
lines changed

‎scripts/distribute/ci_case_auto.sh

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,25 +47,26 @@ function track_case_status() {
4747
cd ${log_path} || { echo "Failed to enter log_path: $log_path"; return 1; }
4848

4949
total_count=$(ls -1 "$prefix"* 2>/dev/null | wc -l)
50-
run_fail_count=$(ls -1 "$prefix"*_FAIL 2>/dev/null | wc -l)
51-
loss_fail_count=$(grep 'check failed! ' result.log | awk -v prefix="$prefix_var" '{if ($2 ~ "^" prefix) print $2}'| wc -l)
50+
run_fail_count=$(ls -1 "$prefix"*_FAIL* 2>/dev/null | wc -l)
51+
loss_fail_count=$(grep 'check failed! ' result.log | awk -v prefix="$prefix" '{if ($2 ~ "^" prefix) print $2}'| wc -l)
5252

53-
# return original path
5453
echo -e "\033[31m ---- $case_name total tests : $total_count \033"
5554
if [ $run_fail_count -eq 0 ] && [ $loss_fail_count -eq 0 ]; then
56-
echo -e "\033[32m ---- $case_name all cases Success \033"
55+
echo -e "\033[32m ---- all cases Success \033"
5756
else
5857
if [[ $run_fail_count -ne 0 ]] ; then
5958
echo -e "\033[31m ---- $case_name runtime failed test : $run_fail_count \033"
60-
ls -1 "$prefix"*_FAIL 2>/dev/null
59+
ls -1 "$prefix"*_FAIL* 2>/dev/null | awk -v OFS="\t" '{print "\t" $0 "(failed)"}'
6160
fi
6261
if [[ $loss_fail_count -ne 0 ]] ; then
63-
echo -e "\033[31m ---- $case_name loss verification failed test : $loss_fail_count \033"
64-
grep 'check failed! ' result.log | awk -v prefix="$prefix_var" '{if ($2 ~ "^" prefix) print $2}'
62+
echo -e "\033[31m ---- $case_name verification failed test : $loss_fail_count \033"
63+
grep 'check failed! ' result.log | awk -v prefix="$prefix" 'BEGIN {OFS="\t"} {if ($2 ~ "^" prefix) print "\t" $2 "(failed)"}'
6564
fi
65+
return 2
6666
fi
6767
cd "$original_path" || { echo "Failed to return to original path: $original_path"; return 1; }
68-
}
68+
return 0
69+
}
6970

7071
# NOTE: Please place the new tests as much as possible after the existing tests
7172
function llama_case_list_auto() {

‎scripts/distribute/ci_case_dy.sh

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,25 +36,26 @@ function track_case_status() {
3636
cd ${log_path} || { echo "Failed to enter log_path: $log_path"; return 1; }
3737

3838
total_count=$(ls -1 "$prefix"* 2>/dev/null | wc -l)
39-
run_fail_count=$(ls -1 "$prefix"*_FAIL 2>/dev/null | wc -l)
40-
loss_fail_count=$(grep 'check failed! ' result.log | awk -v prefix="$prefix_var" '{if ($2 ~ "^" prefix) print $2}'| wc -l)
39+
run_fail_count=$(ls -1 "$prefix"*_FAIL* 2>/dev/null | wc -l)
40+
loss_fail_count=$(grep 'check failed! ' result.log | awk -v prefix="$prefix" '{if ($2 ~ "^" prefix) print $2}'| wc -l)
4141

42-
# return original path
4342
echo -e "\033[31m ---- $case_name total tests : $total_count \033"
4443
if [ $run_fail_count -eq 0 ] && [ $loss_fail_count -eq 0 ]; then
45-
echo -e "\033[32m ---- $case_name all cases Success \033"
44+
echo -e "\033[32m ---- all cases Success \033"
4645
else
4746
if [[ $run_fail_count -ne 0 ]] ; then
4847
echo -e "\033[31m ---- $case_name runtime failed test : $run_fail_count \033"
49-
ls -1 "$prefix"*_FAIL 2>/dev/null
48+
ls -1 "$prefix"*_FAIL* 2>/dev/null | awk -v OFS="\t" '{print "\t" $0 "(failed)"}'
5049
fi
5150
if [[ $loss_fail_count -ne 0 ]] ; then
52-
echo -e "\033[31m ---- $case_name loss verification failed test : $loss_fail_count \033"
53-
grep 'check failed! ' result.log | awk -v prefix="$prefix_var" '{if ($2 ~ "^" prefix) print $2}'
51+
echo -e "\033[31m ---- $case_name verification failed test : $loss_fail_count \033"
52+
grep 'check failed! ' result.log | awk -v prefix="$prefix" 'BEGIN {OFS="\t"} {if ($2 ~ "^" prefix) print "\t" $2 "(failed)"}'
5453
fi
54+
return 2
5555
fi
5656
cd "$original_path" || { echo "Failed to return to original path: $original_path"; return 1; }
57-
}
57+
return 0
58+
}
5859

5960
function gpt_case_list_dygraph(){
6061
# The test name must have "gpt_" as a prefix, which will

‎scripts/distribute/run_ci.sh

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -139,8 +139,16 @@ fi
139139
}
140140
####################################
141141
print_info(){
142-
#解决异常退出-6的问题，CI中的偶现问题，无法复现
143-
if [[ $1 -ne 0 ]] && [[ $1 -ne 250 ]];then
142+
if [ $1 -eq 250 ];then
143+
#解决异常退出-6的问题，CI中的偶现问题，无法复现
144+
echo -e "\033[1;31m"
145+
echo -e "\033[1;31m The CI execution encountered an abnormal termination with error code exit -6. \033[0m"
146+
echo -e "\033[1;31m This is an intermittent issue. \033[0m"
147+
echo -e "\033[1;31m Please re-run the CI. \033[0m"
148+
echo -e "\033[1;31m"
149+
exit 2
150+
fi
151+
if [[ $1 -ne 0 ]];then
144152
EXCODE=2
145153
if [ ! -f ${log_path}/$2 ];then
146154
echo -e "\033[31m run $2 CI FAIL \033"
@@ -151,7 +159,7 @@ if [[ $1 -ne 0 ]] && [[ $1 -ne 250 ]];then
151159
fi
152160
exit $EXCODE
153161
else
154-
echo -e "\033[32m run $3 CI SUCCESS \033"
162+
echo -e "\033[32m The $3 CI has completed \033"
155163
fi
156164
}
157165
####################################
@@ -174,24 +182,25 @@ function track_case_status() {
174182
cd ${log_path} || { echo "Failed to enter log_path: $log_path"; return 1; }
175183

176184
total_count=$(ls -1 "$prefix"* 2>/dev/null | wc -l)
177-
run_fail_count=$(ls -1 "$prefix"*_FAIL 2>/dev/null | wc -l)
178-
loss_fail_count=$(grep 'check failed! ' result.log | awk -v prefix="$prefix_var" '{if ($2 ~ "^" prefix) print $2}'| wc -l)
185+
run_fail_count=$(ls -1 "$prefix"*_FAIL* 2>/dev/null | wc -l)
186+
loss_fail_count=$(grep 'check failed! ' result.log | awk -v prefix="$prefix" '{if ($2 ~ "^" prefix) print $2}'| wc -l)
179187

180-
# return original path
181188
echo -e "\033[31m ---- $case_name total tests : $total_count \033"
182189
if [ $run_fail_count -eq 0 ] && [ $loss_fail_count -eq 0 ]; then
183-
echo -e "\033[32m ---- $case_name all cases Success \033"
190+
echo -e "\033[32m ---- all cases Success \033"
184191
else
185192
if [[ $run_fail_count -ne 0 ]] ; then
186193
echo -e "\033[31m ---- $case_name runtime failed test : $run_fail_count \033"
187-
ls -1 "$prefix"*_FAIL 2>/dev/null
194+
ls -1 "$prefix"*_FAIL* 2>/dev/null | awk -v OFS="\t" '{print "\t" $0 "(failed)"}'
188195
fi
189196
if [[ $loss_fail_count -ne 0 ]] ; then
190-
echo -e "\033[31m ---- $case_name loss verification failed test : $loss_fail_count \033"
191-
grep 'check failed! ' result.log | awk -v prefix="$prefix_var" '{if ($2 ~ "^" prefix) print $2}'
197+
echo -e "\033[31m ---- $case_name verification failed test : $loss_fail_count \033"
198+
grep 'check failed! ' result.log | awk -v prefix="$prefix" 'BEGIN {OFS="\t"} {if ($2 ~ "^" prefix) print "\t" $2 "(failed)"}'
192199
fi
200+
return 2
193201
fi
194202
cd "$original_path" || { echo "Failed to return to original path: $original_path"; return 1; }
203+
return 0
195204
}
196205
####################################
197206
get_diff_TO_case # 获取待执行case列表
@@ -238,6 +247,7 @@ if [[ ${#case_list[*]} -ne 0 ]];then
238247
echo -e "\033[31m ---- end run case \033"
239248

240249
track_case_status $FUNCNAME ""
250+
EXCODE=$?
241251
else
242252
echo -e "\033[32m Changed Not CI case, Skips \033"
243253
EXCODE=0

0 commit comments

Comments
 (0)