From 4b266b5660673655b7e6038ebbcc08c1996f369b Mon Sep 17 00:00:00 2001 From: Marcin Naczk Date: Wed, 19 Jan 2022 18:21:00 +0000 Subject: [PATCH] Upgrade Build IGC in github Action Upgrade Build IGC in github Action: -Adding OpenCL Clang caching -Adding dropping Artifact from ready builds -SPIRV-LLVM-Translator build caching added -add e-mail notification when build fails --- .github/workflows/build-IGC.yml | 154 ++++++++++++++++++++++++++++++-- scripts/Dockerfile-ubuntu1804 | 23 ----- scripts/Dockerfile-ubuntu2004 | 23 ----- scripts/buildIGC.sh | 47 +++++----- scripts/buildSLT.sh | 60 +++++++++++++ 5 files changed, 232 insertions(+), 75 deletions(-) delete mode 100644 scripts/Dockerfile-ubuntu1804 delete mode 100644 scripts/Dockerfile-ubuntu2004 create mode 100644 scripts/buildSLT.sh diff --git a/.github/workflows/build-IGC.yml b/.github/workflows/build-IGC.yml index 08e636848..26bd4c5a2 100644 --- a/.github/workflows/build-IGC.yml +++ b/.github/workflows/build-IGC.yml @@ -1,6 +1,6 @@ #=========================== begin_copyright_notice ============================ # -# Copyright (C) 2021 Intel Corporation +# Copyright (C) 2022 Intel Corporation # # SPDX-License-Identifier: MIT # @@ -13,16 +13,160 @@ on: branches: [ master ] jobs: - build: + download-OpenCL-Clang: # openCL Clang is downloaded from intel/intel-graphics-compiler releases + if: github.repository == 'intel/intel-graphics-compiler' + runs-on: ubuntu-latest + steps: + - uses: oprypin/find-latest-tag@v1 + with: + repository: intel/intel-graphics-compiler + releases-only: true + id: get-latest-IGC-release-tag + - name: caching downloaded OpenCL Clang # Downloading a new OpenCL Clang only after the new IGC is released + id: cache-igc-release + uses: actions/cache@v2 + with: + path: ./igc-official-release + key: ${{ steps.get-latest-IGC-release-tag.outputs.tag }} + - name: creating a folder for downloaded deb files + if: steps.cache-igc-release.outputs.cache-hit != 'true' + run: mkdir igc-official-release + - name: download latest igc release + if: steps.cache-igc-release.outputs.cache-hit != 'true' + working-directory: ./igc-official-release + # To install openCL clang from Deb package, IGC must also be installed. Therefore, the last IGC release is also downloaded. + run: curl -s https://api.github.com/repos/intel/intel-graphics-compiler/releases/latest | grep browser_download_url | egrep 'opencl_|core_' | cut -d '"' -f 4 | wget -qi - + - name: checking if downloaded deb files are here + run: ls ./igc-official-release + + build-SPIRV-LLVM-Translator: if: github.repository == 'intel/intel-graphics-compiler' runs-on: ubuntu-latest strategy: fail-fast: false matrix: - os: [ ubuntu1804, ubuntu2004 ] + ubuntu_version: [ 18, 20 ] + llvm_ver: [ 10, 11 ] + steps: + - uses: actions/checkout@v2 + - name: cloning SPIRV-LLVM-Translator repository + run: git clone --branch llvm_release_${{ matrix.llvm_ver }}0 https://github.com/KhronosGroup/SPIRV-LLVM-Translator + - name: getting latest commit hash from SPIRV-LLVM-Translator repository + id: SPIRV-LLVM-Translator-hash + working-directory: ./SPIRV-LLVM-Translator + run: | + SPIRV_TRANSLATOR_SHA_SHORT=$(git log -1 --pretty=format:"%h") + echo "This is SPIRV-LLVM-Translator repository latest commit hash = ${SPIRV_TRANSLATOR_SHA_SHORT}" + echo "::set-output name=spirv_translator_sha_short::${SPIRV_TRANSLATOR_SHA_SHORT}" + - name: caching SPIRV-LLVM-Translator deb files + id: cache-spirv-llvm-translator + uses: actions/cache@v2 + with: + path: ./build-SPIRV-LLVM-Translator + key: ${{ steps.SPIRV-LLVM-Translator-hash.outputs.spirv_translator_sha_short }}-ubuntu${{ matrix.ubuntu_version }}-llvm${{ matrix.llvm_ver }} + - name: run Docker + if: steps.cache-spirv-llvm-translator.outputs.cache-hit != 'true' + run: | + docker run --name=buildslt \ + -v $(pwd)/scripts/buildSLT.sh:/buildSLT.sh \ + --env DEBIAN_FRONTEND="noninteractive" \ + --env UBUNTU_VERSION=${{ matrix.ubuntu_version }} \ + --env LLVM_VERSION=${{ matrix.llvm_ver }} \ + ubuntu:${{ matrix.ubuntu_version }}.04 \ + sh /buildSLT.sh + - name: creating folders to store SPIRV-LLVM-Translator build outputs + if: steps.cache-spirv-llvm-translator.outputs.cache-hit != 'true' + run: mkdir build-SPIRV-LLVM-Translator && mkdir temp-SPIRV-LLVM-Translator-build + - name: copying artifact from docker to github host + if: steps.cache-spirv-llvm-translator.outputs.cache-hit != 'true' + run: sudo docker cp buildslt:/SPIRV-LLVM-Translator/build ./temp-SPIRV-LLVM-Translator-build + - name: copy deb file to main artifact folder + if: steps.cache-spirv-llvm-translator.outputs.cache-hit != 'true' + run: sudo cp ./temp-SPIRV-LLVM-Translator-build/build/*.deb ./build-SPIRV-LLVM-Translator + + build-IGC: + needs: [build-SPIRV-LLVM-Translator, download-OpenCL-Clang] + if: github.repository == 'intel/intel-graphics-compiler' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ubuntu_version: [ 18, 20 ] llvm_ver: [ 10, 11 ] compiler: [ gcc, clang ] steps: - uses: actions/checkout@v2 - - name: Run Script Selectiong Docker - run: docker build -f scripts/Dockerfile-${{ matrix.os }} --build-arg BUILD_OS=${{ matrix.os }} --build-arg LLVM_VERSION=${{ matrix.llvm_ver }} --build-arg COMPILER=${{ matrix.compiler }} ./scripts + +#Get OpenCL Clang + - uses: oprypin/find-latest-tag@v1 + with: + repository: intel/intel-graphics-compiler + releases-only: true + id: get-latest-IGC-release-tag + - name: getting OpenCL Clang from cache from the previous job + id: cache-igc-release + uses: actions/cache@v2 + with: + path: ./igc-official-release + key: ${{ steps.get-latest-IGC-release-tag.outputs.tag }} + - name: checking if deb files are here + run: ls ./igc-official-release + +#Get SPIRV-LLVM-Translator + - name: cloning SPIRV-LLVM-Translator repository + run: git clone --branch llvm_release_${{ matrix.llvm_ver }}0 https://github.com/KhronosGroup/SPIRV-LLVM-Translator + - name: getting latest commit hash from SPIRV-LLVM-Translator repository + id: SPIRV-LLVM-Translator-hash + working-directory: ./SPIRV-LLVM-Translator + run: | + SPIRV_TRANSLATOR_SHA_SHORT=$(git log -1 --pretty=format:"%h") + echo "This is SPIRV-LLVM-Translator hash = ${SPIRV_TRANSLATOR_SHA_SHORT}" + echo "::set-output name=spirv_translator_sha_short::${SPIRV_TRANSLATOR_SHA_SHORT}" + - name: getting SPIRV-LLVM-Translator deb files from cache from the previous job + id: cache-spirv-llvm-translator + uses: actions/cache@v2 + with: + path: ./build-SPIRV-LLVM-Translator + key: ${{ steps.SPIRV-LLVM-Translator-hash.outputs.spirv_translator_sha_short }}-ubuntu${{ matrix.ubuntu_version }}-llvm${{ matrix.llvm_ver }} + +#Build IGC + - name: run Docker + run: | + docker run --name=buildigc \ + -v $(pwd)/scripts/buildIGC.sh:/buildIGC.sh \ + -v $(pwd)/igc-official-release:/igc-official-release \ + -v $(pwd)/build-SPIRV-LLVM-Translator:/build-SPIRV-LLVM-Translator \ + --env DEBIAN_FRONTEND="noninteractive" \ + --env UBUNTU_VERSION=${{ matrix.ubuntu_version }} \ + --env LLVM_VERSION=${{ matrix.llvm_ver }} \ + --env COMPILER=${{ matrix.compiler }} \ + ubuntu:${{ matrix.ubuntu_version }}.04 \ + sh /buildIGC.sh + - name: creating folder to store output artifacts + run: mkdir Artifacts + - name: copying artifact from docker to github host + run: sudo docker cp buildigc:/workspace/igc/build/DEB-FILES ./Artifacts + - name: get short hash + id: igc-current-short-Hash + run: echo "::set-output name=sha_short::${GITHUB_SHA::7}" + - uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: IGC_Ubuntu${{ matrix.ubuntu_version }}_llvm${{ matrix.llvm_ver }}_${{ matrix.compiler }}-${{ steps.igc-current-short-Hash.outputs.sha_short }} + path: ./Artifacts/DEB-FILES/*.deb + retention-days: 7 + - name: Send mail + if: ${{ failure() }} + uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{secrets.MAIL_ADDRESS}} + password: ${{secrets.MAIL_PASSWORD}} + subject: Github Actions IGC build failed + to: marcin.naczk@intel.com + from: Github Action + secure: true + body: Build Failed. The job ${{github.job}} of ${{github.repository}}--${github.sha} Failed. Run ID ${{github.run_id}}. Run Number ${{github.run_number}} + ignore_cert: true + priority: low diff --git a/scripts/Dockerfile-ubuntu1804 b/scripts/Dockerfile-ubuntu1804 deleted file mode 100644 index 89b4227a8..000000000 --- a/scripts/Dockerfile-ubuntu1804 +++ /dev/null @@ -1,23 +0,0 @@ -#=========================== begin_copyright_notice ============================ -# -# Copyright (C) 2021 Intel Corporation -# -# SPDX-License-Identifier: MIT -# -#============================ end_copyright_notice ============================= - -FROM ubuntu:18.04 -LABEL maintainer="marcin.naczk@intel.com" -ENV DEBIAN_FRONTEND="noninteractive" - -ARG BUILD_OS -ARG LLVM_VERSION -ARG COMPILER - -RUN echo "[Build Status Dockerfile] Dockerfile start working" -RUN echo "[Build Status Dockerfile] BUILD_OS = ${BUILD_OS}" -RUN echo "[Build Status Dockerfile] LLVM_VERSION = ${LLVM_VERSION}" -RUN echo "[Build Status Dockerfile] COMPILER = ${COMPILER}" - -COPY . /home -RUN /bin/bash -c "BUILD_OS=${BUILD_OS} LLVM_VERSION=${LLVM_VERSION} COMPILER=${COMPILER} sh /home/buildIGC.sh" \ No newline at end of file diff --git a/scripts/Dockerfile-ubuntu2004 b/scripts/Dockerfile-ubuntu2004 deleted file mode 100644 index 5a61e61c7..000000000 --- a/scripts/Dockerfile-ubuntu2004 +++ /dev/null @@ -1,23 +0,0 @@ -#=========================== begin_copyright_notice ============================ -# -# Copyright (C) 2021 Intel Corporation -# -# SPDX-License-Identifier: MIT -# -#============================ end_copyright_notice ============================= - -FROM ubuntu:20.04 -LABEL maintainer="marcin.naczk@intel.com" -ENV DEBIAN_FRONTEND="noninteractive" - -ARG BUILD_OS -ARG LLVM_VERSION -ARG COMPILER - -RUN echo "[Build Status Dockerfile] Dockerfile start working" -RUN echo "[Build Status Dockerfile] BUILD_OS = ${BUILD_OS}" -RUN echo "[Build Status Dockerfile] LLVM_VERSION = ${LLVM_VERSION}" -RUN echo "[Build Status Dockerfile] COMPILER = ${COMPILER}" - -COPY . /home -RUN /bin/bash -c "BUILD_OS=${BUILD_OS} LLVM_VERSION=${LLVM_VERSION} COMPILER=${COMPILER} sh /home/buildIGC.sh" \ No newline at end of file diff --git a/scripts/buildIGC.sh b/scripts/buildIGC.sh index 6d67c7b6d..5b629961a 100644 --- a/scripts/buildIGC.sh +++ b/scripts/buildIGC.sh @@ -2,26 +2,26 @@ #=========================== begin_copyright_notice ============================ # -# Copyright (C) 2021 Intel Corporation +# Copyright (C) 2022 Intel Corporation # # SPDX-License-Identifier: MIT # #============================ end_copyright_notice ============================= set -e -# BUILD_OS supported value [ ubuntu1804, ubuntu2004 ] default ubuntu2004 +# UBUNTU_VERSION supported value [ 18, 20 ] default 20 # LLVM_VERSION supported value [ 10, 11, 12 ] default 11 # COMPILER supported value [ gcc, clang ] default gcc # OWN_CMAKE_FLAGS not suported but can be use as WA (each flag should be with -D prefix) default empty -# example run: BUILD_OS=ubuntu2004 LLVM_VERSION=11 COMPILER=gcc sh /home/buildIGC.sh +# example run: UBUNTU_VERSION=ubuntu2004 LLVM_VERSION=11 COMPILER=gcc sh /home/buildIGC.sh echo "====================BUILD IGC=========================" echo "[Build Status] build script started" -if [ -z ${BUILD_OS+x} ]; then - echo "[Build Status] BUILD_OS is unset, use default ubuntu2004"; - BUILD_OS="ubuntu2004" +if [ -z ${UBUNTU_VERSION+x} ]; then + echo "[Build Status] UBUNTU_VERSION is unset, use default 20"; + UBUNTU_VERSION="20" else - echo "[Build Status] BUILD_OS = ${BUILD_OS}" + echo "[Build Status] UBUNTU_VERSION = ${UBUNTU_VERSION}" fi if [ -z ${LLVM_VERSION+x} ]; then echo "[Build Status] LLVM_VERSION is unset, use default 11"; @@ -45,10 +45,9 @@ fi apt-get update apt-get install -y flex bison libz-dev cmake curl wget build-essential git software-properties-common unzip -apt-get update echo "[Build Status] flex bison libz-dev cmake curl wget build-essential git software-properties-common INSTALLED" -if [ "$BUILD_OS" = "ubuntu1804" ]; then +if [ "$UBUNTU_VERSION" = "18" ]; then wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - apt-add-repository "deb https://apt.kitware.com/ubuntu/ bionic main" apt update @@ -62,25 +61,26 @@ fi apt-get install -y llvm-"$LLVM_VERSION" llvm-"$LLVM_VERSION"-dev clang-"$LLVM_VERSION" liblld-"$LLVM_VERSION" liblld-"$LLVM_VERSION"-dev echo "[Build Status] LLVM INSTALLED" -if [ "$BUILD_OS" = "ubuntu1804" ] && [ "$LLVM_VERSION" = "11" ]; then +if [ "$UBUNTU_VERSION" = "18" ] && [ "$LLVM_VERSION" = "11" ]; then LLVM_VERSION_PREFERRED="$LLVM_VERSION".1.0 else LLVM_VERSION_PREFERRED="$LLVM_VERSION".0.0 fi echo "[Build Status] LLVM_VERSION_PREFERRED = $LLVM_VERSION_PREFERRED" +echo "[Build Status] Prepare install OpenCL Clang" +dpkg -i ./igc-official-release/*.deb +echo "[Build Status] OpenCL Clang installed" + + +echo "[Build Status] Install SPIRV-LLVM-Translator" +dpkg -i ./build-SPIRV-LLVM-Translator/*.deb +mv /usr/lib/libLLVMSPIRVLib.a /usr/local/lib/libLLVMSPIRVLib.a # WA Cpack wrongly pack deb file +echo "[Build Status] SPIRV-LLVM-Translator INSTALLED" + mkdir workspace cd workspace -echo "[Build Status] build and install SPIRV-LLVM-Translator" -/usr/bin/git clone --branch llvm_release_"$LLVM_VERSION"0 https://github.com/KhronosGroup/SPIRV-LLVM-Translator -mkdir SPIRV-LLVM-Translator/build && cd SPIRV-LLVM-Translator/build -cmake .. -DBASE_LLVM_VERSION="$LLVM_VERSION_PREFERRED" -make llvm-spirv -j`nproc` -make install -cd ../.. -echo "[Build Status] SPIRV-LLVM-Translator INSTALLED" - /usr/bin/git version /usr/bin/git clone https://github.com/intel/intel-graphics-compiler ./igc cd igc @@ -90,11 +90,6 @@ echo "[Build Status] IGC commit hash below:" /usr/bin/git clone https://github.com/KhronosGroup/SPIRV-Headers.git ../SPIRV-Headers /usr/bin/git clone https://github.com/KhronosGroup/SPIRV-Tools.git ../SPIRV-Tools echo "[Build Status] All necessary repository CLONED" -mkdir build -cd build -curl -s https://api.github.com/repos/intel/intel-graphics-compiler/releases/latest | grep browser_download_url | egrep 'opencl_|core_' | cut -d '"' -f 4 | wget -qi - -dpkg -i *.deb -echo "[Build Status] Old IGC with opencl-clang downloaded and INSTALLED, WA to install opencl-clang" CONFIG_VARS="-DIGC_OPTION__LLVM_MODE=Prebuilds -DIGC_OPTION__LLVM_PREFERRED_VERSION=$LLVM_VERSION_PREFERRED" case $COMPILER in @@ -105,9 +100,13 @@ esac CONFIG_VARS="$CONFIG_VARS $OWN_CMAKE_FLAGS" echo "[Build Status] CONFIG_VARS = $CONFIG_VARS" +mkdir build && cd build cmake ../ $CONFIG_VARS echo "[Build Status] Cmake created" make -j`nproc` echo "[Build Status] make DONE" +cpack +mkdir DEB-FILES && mv ./*.deb ./DEB-FILES +echo "[Build Status] cpack DONE" echo "====================BUILD IGC=========================" diff --git a/scripts/buildSLT.sh b/scripts/buildSLT.sh new file mode 100644 index 000000000..e1a6a2c3f --- /dev/null +++ b/scripts/buildSLT.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +#=========================== begin_copyright_notice ============================ +# +# Copyright (C) 2022 Intel Corporation +# +# SPDX-License-Identifier: MIT +# +#============================ end_copyright_notice ============================= + +set -e +# UBUNTU_VERSION supported value [ 18, 20 ] default 20 +# LLVM_VERSION supported value [ 10, 11, 12 ] default 11 +# OWN_CMAKE_FLAGS not suported but can be use as WA (each flag should be with -D prefix) default empty +# example run: UBUNTU_VERSION=ubuntu2004 LLVM_VERSION=11 sh /home/buildSLT.sh + +echo "====================BUILD SPIRV-LLVM-Translator=========================" +echo "[Build Status] build script started" +if [ -z ${UBUNTU_VERSION+x} ]; then + echo "[Build Status] UBUNTU_VERSION is unset, use default 20"; + UBUNTU_VERSION="20" +else + echo "[Build Status] UBUNTU_VERSION = ${UBUNTU_VERSION}" +fi +if [ -z ${LLVM_VERSION+x} ]; then + echo "[Build Status] LLVM_VERSION is unset, use default 11"; + LLVM_VERSION="11" +else + echo "[Build Status] LLVM_VERSION = ${LLVM_VERSION}" +fi + +apt-get update +apt-get install -y cmake build-essential git +echo "[Build Status] cmake build-essential git INSTALLED" + +if [ "$UBUNTU_VERSION" = "18" ]; then + wget --no-check-certificate -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - + add-apt-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-$LLVM_VERSION main" + apt update + echo '[Build Status] LLVM "$LLVM_VERSION" on Ubuntu18.04 PREPARED' +fi +apt-get install -y llvm-"$LLVM_VERSION" llvm-"$LLVM_VERSION"-dev clang-"$LLVM_VERSION" liblld-"$LLVM_VERSION" liblld-"$LLVM_VERSION"-dev +echo "[Build Status] LLVM INSTALLED" + +if [ "$UBUNTU_VERSION" = "18" ] && [ "$LLVM_VERSION" = "11" ]; then + LLVM_VERSION_PREFERRED="$LLVM_VERSION".1.0 +else + LLVM_VERSION_PREFERRED="$LLVM_VERSION".0.0 +fi +echo "[Build Status] LLVM_VERSION_PREFERRED = $LLVM_VERSION_PREFERRED" + +echo "[Build Status] build and install SPIRV-LLVM-Translator" +/usr/bin/git clone --branch llvm_release_"$LLVM_VERSION"0 https://github.com/KhronosGroup/SPIRV-LLVM-Translator +cd SPIRV-LLVM-Translator +echo 'set(CPACK_GENERATOR "DEB")' >> CMakeLists.txt && echo 'set(CPACK_DEBIAN_PACKAGE_MAINTAINER "David Doria") # required' >> CMakeLists.txt && echo 'include(CPack)' >> CMakeLists.txt +mkdir build && cd build +cmake .. -DBASE_LLVM_VERSION="$LLVM_VERSION_PREFERRED" +make llvm-spirv -j`nproc` +cpack +echo "[Build Status] SPIRV-LLVM-Translator Packed"