Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into accuracy_embedding

Author: ooooo-create

.github/workflows/CI.yml

@@ -110,7 +110,8 @@ jobs:
   slice:
     name: Slice
     uses: ./.github/workflows/_Slice.yml
-    needs: [build-docker, build]
+    needs: [clone, build-docker, build]
     with:
       can-skip: ${{ needs.build.outputs.can-skip }}
       docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+      slice-check: ${{ needs.clone.outputs.slice-check }}

.github/workflows/Coverage.yml

@@ -140,7 +140,7 @@ jobs:
           git checkout test
           echo "Pull upstream $BRANCH"
           source ${{ github.workspace }}/../../../proxy
-          git pull upstream $BRANCH --no-edit
+          bash ci/git_pull.sh $BRANCH
           '
 
       - name: Check bypass
@@ -203,7 +203,7 @@ jobs:
         if: steps.check-bypass.outputs.can-skip != 'true'
         env:
           home_path: ${{ github.workspace }}/..
-          bos_file: ${{ github.workspace }}/../bos/BosClient.py
+          bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
           paddle_whl: paddlepaddle_gpu-0.0.0-cp39-cp39-linux_x86_64.whl
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
@@ -214,9 +214,9 @@ jobs:
           export AK=paddle
           export SK=paddle
           if [ ! -f "${{ env.bos_file }}" ]; then
-            wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
-            mkdir ${{ env.home_path }}/bos
-            tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
+            wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
+            mkdir ${{ env.home_path }}/bos_retry
+            tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
           fi
           cd /paddle/dist
           echo "Uploading coverage build size"
@@ -330,7 +330,7 @@ jobs:
           source ${{ github.workspace }}/../../../proxy
           git checkout test
           echo "Pull upstream develop"
-          git pull upstream $BRANCH --no-edit
+          bash ci/git_pull.sh $BRANCH
           '
 
       - name: Test

.github/workflows/_Clone-linux.yml

@@ -13,6 +13,8 @@ on:
     outputs:
       can-skip:
         value: ${{ jobs.clone.outputs.can-skip }}
+      slice-check:
+        value: ${{ jobs.clone.outputs.slice-check }}
 
 permissions: read-all
 
@@ -31,6 +33,7 @@ jobs:
     name: Clone Paddle
     outputs:
       can-skip: ${{ steps.check-bypass.outputs.can-skip }}
+      slice-check: ${{ steps.check-execution.outputs.slice-check }}
 
     if: github.repository_owner == 'PaddlePaddle'
     runs-on:
@@ -45,6 +48,7 @@ jobs:
           fetch-depth: 1000
 
       - name: Merge PR to test branch
+        id: check-execution
         run: |
           git config --unset http.https://github.com/.extraheader
           git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'"
@@ -60,6 +64,7 @@ jobs:
           git fetch origin pull/${{ github.event.pull_request.number }}/head:pr
           git merge --no-ff pr
           git branch -d pr
+          source ${ci_scripts}/check_execution.sh
           bash ${ci_scripts}/third_party_tag.sh
 
       - name: Check bypass

.github/workflows/_Distribute-stable.yml

@@ -136,7 +136,7 @@ jobs:
           source ${{ github.workspace }}/../../../proxy
           git checkout test
           echo "Pull upstream develop"
-          git pull upstream $BRANCH --no-edit
+          bash ci/git_pull.sh $BRANCH
           '
 
       - name: Check bypass
@@ -178,10 +178,7 @@ jobs:
             md5sum fa_libs.tar |awk "{print \$1}" >MD5.txt
             cd .. && tar -zcf flashattn_libs_${{ env.flashattn_version }}.tar ./flashattn_libs_${{ env.flashattn_version }}
           fi
-          cd ${work_dir}/build
-          rm -rf $(find . -name "*.a")
-          rm -rf $(find . -name "*.o")
-          cd ${work_dir}/build/third_party && find `ls | grep -v "dlpack"| grep -v "install"|grep -v "eigen3"|grep -v "gflags" ` -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
+          bash ${ci_scripts}/compress_build.sh
           cd ${work_dir}/..
           tar --use-compress-program="pzstd -1" --warning=no-file-changed -cf Paddle.tar.gz paddle
           '
@@ -190,7 +187,7 @@ jobs:
         if: steps.check-bypass.outputs.can-skip != 'true'
         env:
           home_path: ${{ github.workspace }}/..
-          bos_file: ${{ github.workspace }}/../bos/BosClient.py
+          bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
           paddle_whl: paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
@@ -202,9 +199,9 @@ jobs:
           python -m pip install bce-python-sdk==0.8.74
           echo "::endgroup::"
           if [ ! -f "${{ env.bos_file }}" ]; then
-            wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
-            mkdir ${{ env.home_path }}/bos
-            tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
+            wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
+            mkdir ${{ env.home_path }}/bos_retry
+            tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
           fi
           cd ..
           source ${{ github.workspace }}/../../../unproxy
@@ -327,7 +324,7 @@ jobs:
           git checkout test
           echo "Pull upstream $BRANCH"
           source ${{ github.workspace }}/../../../proxy
-          git pull upstream $BRANCH --no-edit
+          bash ci/git_pull.sh $BRANCH
           '
 
       - name: Test

.github/workflows/_Inference.yml

@@ -124,7 +124,7 @@ jobs:
           git config pull.rebase false
           git checkout test
           echo "Pull upstream develop"
-          git pull upstream $BRANCH --no-edit
+          bash ci/git_pull.sh $BRANCH
           '
 
       - name: Run build
@@ -140,19 +140,20 @@ jobs:
         if: steps.check-bypass.outputs.can_skip != 'true'
         env:
           home_path: /action-runner
-          bos_file: /action-runner/bos/BosClient.py
+          bos_file: /action-runner/bos_retry/BosClient.py
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           echo "::group::Install bce-python-sdk"
           source /action-runner/proxy
           python -m pip install bce-python-sdk==0.8.74
           echo "::endgroup::"
+          rm -rf `find . -name "*.o"`
           export AK=paddle
           export SK=paddle
           if [ ! -f "${{ env.bos_file }}" ]; then
-            wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
-            mkdir ${{ env.home_path }}/bos
-            tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
+            wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
+            mkdir ${{ env.home_path }}/bos_retry
+            tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
           fi
           source /action-runner/unproxy
           cd /

.github/workflows/_Linux-CPU.yml

@@ -105,8 +105,9 @@ jobs:
           git config --global user.email "paddle_ci@example.com"
           git remote add upstream https://github.com/PaddlePaddle/Paddle.git
           source ${{ github.workspace }}/../../../proxy
+          cp ci/git_pull.sh ci/git_pull_copy.sh
           git checkout $BRANCH
-          git pull upstream $BRANCH
+          bash ci/git_pull_copy.sh $BRANCH
           git fetch upstream $BRANCH
           git checkout test
           git merge --no-edit $BRANCH

.github/workflows/_Linux-NPU.yml

@@ -49,7 +49,7 @@ jobs:
           git checkout test
           echo "Pull upstream develop"
           source ${{ github.workspace }}/../../../proxy
-          git pull upstream $BRANCH --no-edit
+          bash ci/git_pull.sh $BRANCH
 
       - name: Determine the runner
         run: |

.github/workflows/_Linux-XPU.yml

@@ -122,7 +122,7 @@ jobs:
           git config pull.rebase false
           git checkout test
           echo "Pull upstream develop or target branch"
-          git pull upstream $BRANCH --no-edit
+          bash ci/git_pull.sh $BRANCH
           '
 
       - name: Run build
@@ -136,8 +136,7 @@ jobs:
           bash ${ci_scripts}/run_setup.sh bdist_wheel
           EXCODE=$?
           python -m pip install bce-python-sdk==0.8.74
-          rm -rf `find . -name "*.a"`
-          rm -rf `find . -name "*.o"`
+          bash ${ci_scripts}/compress_build.sh
           exit $EXCODE
           '
 

.github/workflows/_Linux-build.yml

@@ -132,8 +132,9 @@ jobs:
           git config --global user.email "paddle_ci@example.com"
           git remote add upstream https://github.com/PaddlePaddle/Paddle.git
           source ${{ github.workspace }}/../../../proxy
+          cp ci/git_pull.sh ci/git_pull_copy.sh
           git checkout $BRANCH
-          git pull upstream $BRANCH
+          bash ci/git_pull_copy.sh $BRANCH
           git fetch upstream $BRANCH
           git checkout test
           git merge --no-edit $BRANCH
@@ -173,6 +174,7 @@ jobs:
           paddle_whl: paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
+          bash ${ci_scripts}/compress_build.sh
           export AK=paddle
           export SK=paddle
           if [ ! -f "${{ env.bos_file }}" ]; then

.github/workflows/_Mac.yml

@@ -58,8 +58,9 @@ jobs:
           git config --global user.name "PaddleCI"
           git config --global user.email "paddle_ci@example.com"
           git remote add upstream https://github.com/PaddlePaddle/Paddle.git
+          cp ci/git_pull.sh ci/git_pull_copy.sh
           git checkout $BRANCH
-          git pull upstream $BRANCH
+          bash ci/git_pull_copy.sh $BRANCH
           git fetch upstream $BRANCH
           git checkout test
           git merge --no-edit $BRANCH

.github/workflows/_SOT.yml

@@ -110,8 +110,9 @@ jobs:
           git config --global user.email "paddle_ci@example.com"
           git remote add upstream https://github.com/PaddlePaddle/Paddle.git
           source ${{ github.workspace }}/../../../proxy
+          cp ci/git_pull.sh ci/git_pull_copy.sh
           git checkout $BRANCH
-          git pull upstream $BRANCH
+          bash ci/git_pull_copy.sh $BRANCH
           git fetch upstream $BRANCH
           git checkout test
           git merge --no-edit $BRANCH

.github/workflows/_Slice.yml

@@ -9,6 +9,9 @@ on:
       can-skip:
         type: string
         required: false
+      slice-check:
+        type: string
+        required: false
 
 env:
   PR_ID: ${{ github.event.pull_request.number }}
@@ -28,7 +31,7 @@ defaults:
 jobs:
   slice:
     name: Slice test
-    if: ${{ inputs.can-skip != 'true' }}
+    if: ${{ inputs.can-skip != 'true' && inputs.slice-check == 'true' }}
     runs-on:
       group: slice
     steps:

ci/check_execution.sh

@@ -0,0 +1,20 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+COMMIT_MESSAGE=$(git log -1 --pretty=%B)
+if [[ "$COMMIT_MESSAGE" == *"slice-check"* ]]; then
+    echo "slice-check=true"
+    echo "slice-check=true" >> $GITHUB_OUTPUT
+fi
+git checkout test

ci/compress_build.sh

@@ -0,0 +1,24 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cd ${work_dir}/build
+find ./ -type f -size +200M | xargs du -lh
+rm -rf $(find . -name "*.a")
+rm -rf $(find . -name "*.o")
+rm -rf paddle_inference_install_dir
+rm -rf paddle_inference_c_install_dir
+rm -rf lib.linux-x86_64-310
+find ./ -name "eager_generator" -or -name  "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf
+rm -rf ./python/build/lib.linux-x86_64-310/
+cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete

ci/git_pull.sh

@@ -0,0 +1,30 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BRANCH=$1
+
+MAX_RETRY=5
+n=0
+
+while [ "$n" -lt "$MAX_RETRY" ]; do
+  git pull upstream "$BRANCH" --no-edit && break
+  n=$((n+1))
+  echo "git pull failed, retrying in 5 seconds... ($n/$MAX_RETRY)"
+  sleep 5
+done
+
+if [ "$n" -eq "$MAX_RETRY" ]; then
+  echo "Pull failed after $MAX_RETRY retries"
+  exit 2
+fi

cmake/cudnn.cmake

@@ -17,13 +17,9 @@ endif()
 
 find_path(
   CUDNN_INCLUDE_DIR cudnn.h
-  PATHS ${CUDNN_ROOT}
-        ${CUDNN_ROOT}/include
-        ${CUDNN_ROOT}/include/${TARGET_ARCH}-linux-gnu
-        $ENV{CUDNN_ROOT}
-        $ENV{CUDNN_ROOT}/include
-        ${CUDA_TOOLKIT_INCLUDE}
-        /usr/local/lib/python${PY_VERSION}/dist-packages/nvidia/cudnn/include/
+  PATHS ${CUDNN_ROOT} ${CUDNN_ROOT}/include
+        ${CUDNN_ROOT}/include/${TARGET_ARCH}-linux-gnu $ENV{CUDNN_ROOT}
+        $ENV{CUDNN_ROOT}/include ${CUDA_TOOLKIT_INCLUDE}
   NO_DEFAULT_PATH)
 
 get_filename_component(__libpath_hist ${CUDA_CUDART_LIBRARY} PATH)
@@ -66,7 +62,6 @@ find_library(
   CUDNN_LIBRARY
   NAMES ${CUDNN_LIB_NAME} # libcudnn_static.a
   PATHS ${CUDNN_CHECK_LIBRARY_DIRS} ${CUDNN_INCLUDE_DIR} ${__libpath_hist}
-        /usr/local/lib/python${PY_VERSION}/dist-packages/nvidia/cudnn/lib/
   NO_DEFAULT_PATH
   DOC "Path to cuDNN library.")
 

cmake/external/nvshmem.cmake

@@ -55,9 +55,9 @@ endif()
 
 set(NVSHMEM_PATCH_PATH ${PADDLE_SOURCE_DIR}/patches/nvshmem/nvshmem.patch)
 set(NVSHMEM_PATCH_COMMAND
-    git init && git config user.name "PaddlePaddle" && git config user.email
-    "paddle@baidu.com" && git config --global --add safe.directory
-    ${NVSHMEM_SOURCE_DIR} && git add . && git commit -m "init" && git apply
+    git init && git config --global --add safe.directory ${NVSHMEM_SOURCE_DIR}
+    && git config user.name "PaddlePaddle" && git config user.email
+    "paddle@baidu.com" && git add . && git commit -m "init" && git apply
     ${NVSHMEM_PATCH_PATH})
 
 set(NVSHMEM_LIB ${NVSHMEM_INSTALL_DIR}/lib/libnvshmem.a)