From 79e9b0762c49c2222ea023519ca8b26894242e8e Mon Sep 17 00:00:00 2001 From: "Markus F.X.J. Oberhumer" Date: Wed, 17 Aug 2022 18:05:42 +0200 Subject: [PATCH] CI: work on new cmake-based CI; updates --- .../bug_report.md} | 12 ++ .github/ISSUE_TEMPLATE/config.yml | 1 + .github/ISSUE_TEMPLATE/feature_request.md | 45 ++++++ .github/dependabot.yml | 8 + .github/pull_request_template.md | 14 ++ .github/security.md | 15 ++ .github/travis_testsuite_1.sh | 11 +- .github/workflows/ci.yml | 151 +++++++++++++++--- .github/workflows/close-stale-issues.yml | 12 +- .github/workflows/nopr.yml | 16 ++ src/stub/scripts/upx-clang-format | 18 ++- 11 files changed, 267 insertions(+), 36 deletions(-) rename .github/{ISSUE_TEMPLATE.md => ISSUE_TEMPLATE/bug_report.md} (83%) create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/dependabot.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/security.md create mode 100644 .github/workflows/nopr.yml diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE/bug_report.md similarity index 83% rename from .github/ISSUE_TEMPLATE.md rename to .github/ISSUE_TEMPLATE/bug_report.md index c7122c9f..dde0822a 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,3 +1,15 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +This issue tracker is **ONLY** used for reporting bugs. +Please use [stackoverflow](https://stackoverflow.com) for supporting issues. + + +## What's the problem (or question)? + + + +## What should have happened? + + + +## Do you have an idea for a solution? + + + +## How can we reproduce the issue? + +1. +2. +3. +4. + +## Please tell us details about your environment. + +* UPX version used (`upx --version`): +* Host Operating System and version: +* Host CPU architecture: +* Target Operating System and version: +* Target CPU architecture: diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..668f3226 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 + +updates: + - package-ecosystem: 'github-actions' + directory: '/' + schedule: + # Check for updates to GitHub Actions every weekday + interval: 'daily' diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..6b0a44e6 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,14 @@ +UPX PULL REQUEST NOTES +====================== + +Handling pull requests is actually quite time consuming, so please + +- if you want to contribute **a real C++ code bug-fix** then open an issue +on the main UPX issue tracker first + +- if you want to contribute **a new feature** then by all means open an issue +on the main UPX issue tracker first before starting any coding! + +- please refuse the temptation to "improve" the docs, scripts, CI, makefiles, +cmake build system, spelling errors, etc - we will NOT merge this; only open +an issue if you're sure there is a **real bug** diff --git a/.github/security.md b/.github/security.md new file mode 100644 index 00000000..38e8545d --- /dev/null +++ b/.github/security.md @@ -0,0 +1,15 @@ +# Security Context + +IMPORTANT NOTE: UPX inherits the security context of any files it handles. + +This means that packing, unpacking, or even testing or listing a file requires +the same security considerations as actually executing the file. + +Use UPX on trusted files only! + +## Supported Versions + +| Version | Supported | +| --------- | ------------------ | +| git HEAD | :white_check_mark: | +| any other | :x: | diff --git a/.github/travis_testsuite_1.sh b/.github/travis_testsuite_1.sh index f5dfbe80..f0c02469 100644 --- a/.github/travis_testsuite_1.sh +++ b/.github/travis_testsuite_1.sh @@ -22,13 +22,20 @@ fi if [[ $BM_X == rebuild-stubs ]]; then exit 0 fi + +# convenience [[ -f $upx_exe ]] && upx_exe=$(readlink -en -- "$upx_exe") # create dirs cd / || exit 1 -mkbuilddirs $upx_testsuite_BUILDDIR +if [[ ! -d "$upx_testsuite_SRCDIR/files/packed" ]]; then + echo 'invalid or missing $upx_testsuite_SRCDIR:' + echo ' please git clone https://github.com/upx/upx-testsuite' + echo ' and set (export) the envvar upx_testsuite_SRCDIR to the local file path' + exit 1 +fi +mkbuilddirs "$upx_testsuite_BUILDDIR" cd / && cd "$upx_testsuite_BUILDDIR" || exit 1 -if [[ ! -d $upx_testsuite_SRCDIR/files/packed ]]; then exit 1; fi # /*********************************************************************** # // support functions diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 376db0c4..17120fa5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,9 @@ jobs: run: | uname -a; pwd; id; umask cd ..; mkdir -p deps; cd deps; mkdir packages + # for ubuntu-22.04: install python2 + ##export DEBIAN_FRONTEND=noninteractive + ##sudo apt-get install -y --no-install-recommends python2-minimal # manually install compat libs from Ubuntu 16.04 wget -q 'http://mirror.enzu.com/ubuntu/pool/main/g/gmp/libgmp10_6.1.0+dfsg-2_amd64.deb' wget -q 'http://mirror.enzu.com/ubuntu/pool/main/m/mpfr4/libmpfr4_3.1.4-1_amd64.deb' @@ -30,13 +33,111 @@ jobs: sudo ldconfig wget -q -O - https://github.com/upx/upx-stubtools/releases/download/v20210104/bin-upx-20210104.tar.xz | tar -xJ - name: 'Check out code' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: { submodules: true } - name: 'Rebuild and verify stubs' run: | env C=gcc X=rebuild-stubs TRAVIS_OS_NAME=linux bash ./.github/travis_build.sh + - name: 'Check source code formatting' + run: | + bash ./misc/scripts/check_whitespace_git.sh + env UPX_CLANG_FORMAT="$PWD/../deps/bin-upx-20210104/clang-format-10.0.1" make -C src clang-format + if ! git diff --quiet; then git diff; exit 1; fi + + job-linux-cmake: + needs: [ job-rebuild-and-verify-stubs ] + name: ${{ format('{0} cmake', matrix.os) }} + runs-on: ${{ matrix.os }} + + env: + MATRIX_OS: ${{ matrix.os }} + UPX_CMAKE_BUILD_FLAGS: --verbose + + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-20.04 } + - { os: ubuntu-22.04 } + + steps: + - name: 'Check out code' + uses: actions/checkout@v3 + with: { submodules: true } + - name: 'Build cmake debug-gcc' + run: 'make build/debug-gcc' + - name: 'Build cmake release-gcc' + run: 'make build/release-gcc' + - name: 'Build cmake debug-clang' + run: 'make build/debug-clang' + - name: 'Build cmake release-clang' + run: 'make build/release-clang' + - name: 'Make artifact' + id: make_artifact + run: | + N=upx-${GITHUB_REF_NAME}-${GITHUB_SHA:0:7}-${MATRIX_OS} + mkdir -p "tmp/artifact/$N" + (cd build && cp -ai --parents */upx "../tmp/artifact/$N") + (cd tmp/artifact && tar --sort=name -czf "$N.tar.gz" "$N" && rm -rf "./$N") + # GitHub Actions magic: set "upload_name" Javascript outputs value for use in next step + echo "::set-output name=upload_name::$N" + - name: 'Upload artifact' + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.make_artifact.outputs.upload_name }} + path: tmp/artifact + + job-windows-cmake: + needs: [ job-rebuild-and-verify-stubs ] + name: ${{ format('{0} cmake', matrix.os) }} + runs-on: ${{ matrix.os }} + + env: + MATRIX_OS: ${{ matrix.os }} + UPX_CMAKE_BUILD_FLAGS: --verbose + + strategy: + fail-fast: false + matrix: + include: + # windows-2019 used to work but got broken with the 20220821.1 + # runner-image update; disable for now + # see https://github.com/actions/runner-images.git + ####- { os: windows-2019 } + - { os: windows-2022 } + + steps: + - name: 'Check out code' + uses: actions/checkout@v3 + with: { submodules: true } + - name: 'Inspect runner-image settings' + # see https://github.com/actions/runner-images.git + if: ${{ false }} + run: | + Get-Command bash; Get-Command cmake; Get-Command make + bash --version; cmake --version; make --version + Get-Command cl -ErrorAction SilentlyContinue + - name: 'Build cmake debug' + run: 'make build/debug' + - name: 'Build cmake release' + run: 'make build/release' + - name: 'Make artifact' + id: make_artifact + shell: bash + run: | + N=upx-${GITHUB_REF_NAME}-${GITHUB_SHA:0:7}-${MATRIX_OS} + mkdir -p "tmp/artifact/$N" + (cd build && cp -ai --parents */*/upx.exe "../tmp/artifact/$N") + # GitHub Actions magic: set "upload_name" Javascript outputs value for use in next step + echo "::set-output name=upload_name::$N" + - name: 'Upload artifact' + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.make_artifact.outputs.upload_name }} + path: tmp/artifact job-linux-toolchains: + if: ${{ false }} # disable for now needs: [ job-rebuild-and-verify-stubs ] name: ${{ matrix.name }} runs-on: ${{ matrix.os }} @@ -69,7 +170,7 @@ jobs: fi - name: 'Check out code' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: { submodules: true } - name: 'Prepare sources' @@ -96,12 +197,13 @@ jobs: cp ../build/github/$C/$B/upx/upx.out tmp/upx - name: 'Upload artifact' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ matrix.name }} path: tmp job-windows-cross-toolchains: + if: ${{ false }} # disable for now needs: [ job-rebuild-and-verify-stubs ] name: ${{ matrix.name }} runs-on: ${{ matrix.os }} @@ -122,7 +224,7 @@ jobs: sudo apt-get install -y binutils-mingw-w64-x86-64 g++-mingw-w64-x86-64 wine wine64 - name: 'Check out code' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: { submodules: true } - name: 'Prepare sources' @@ -153,7 +255,7 @@ jobs: cp ../build/github/$CROSS-$C/$B/upx/upx.exe tmp/upx.exe - name: 'Upload artifact' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ matrix.name }} path: tmp @@ -167,6 +269,7 @@ jobs: C: ${{ matrix.C }} B: release H: 'd:\a\upx' + MATRIX_NAME: ${{ matrix.name }} strategy: fail-fast: false @@ -177,7 +280,7 @@ jobs: steps: - name: 'Check out code' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: { submodules: true } - name: 'Prepare sources' @@ -189,13 +292,7 @@ jobs: cd %H% md build build\%C% deps cd deps - curl -sS -L --retry 5 -o ucl-1.03.tar.gz "https://github.com/upx/upx/releases/download/v3.00/ucl-1.03.tar.gz" - curl -sS -L --retry 5 -o zlib-1.2.11.tar.gz "https://github.com/upx/upx/releases/download/v3.00/zlib-1.2.11.tar.gz" git clone --depth=1 https://github.com/upx/upx-testsuite - tar -xzf ucl-1.03.tar.gz - tar -xzf zlib-1.2.11.tar.gz - @REM patch UCL for newer msvc (vsnprintf issue) - sed -i '/# *define.*snprintf *_v*snprintf$/d' ucl-1.03/acc/acc_auto.h - name: 'Set up Developer Command Prompt' uses: ilammy/msvc-dev-cmd@v1 @@ -208,24 +305,24 @@ jobs: @REM setup directories following the VPATH build in travis_build.sh where cl & where link set BDIR=%H%\build\%C%\%B% - md %BDIR% %BDIR%\ucl-1.03 %BDIR%\upx %BDIR%\upx-testsuite %BDIR%\zlib-1.2.11 - set DEFS=-D_CRT_SECURE_NO_WARNINGS + md %BDIR% %BDIR%\ucl %BDIR%\upx %BDIR%\upx-testsuite %BDIR%\zlib + set DEFS=-D_CRT_NONSTDC_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS git rev-parse --short=12 HEAD > %BDIR%\upx\.GITREV.txt @REM build UCL - cd %BDIR%\ucl-1.03 - set s=%H%\deps\ucl-1.03 + cd %BDIR%\ucl + set s=%H%\upx\vendor\ucl cl -MT -O2 -W4 %DEFS% -I%s%\include -I%s% -c %s%\src\*.c link -lib -out:ucl.lib *.obj @REM build zlib - cd %BDIR%\zlib-1.2.11 - cl -MT -O2 -W2 %DEFS% -c %H%\deps\zlib-1.2.11\*.c + cd %BDIR%\zlib + cl -MT -O2 -W3 %DEFS% -c %H%\upx\vendor\zlib\*.c link -lib -out:zlib.lib *.obj @REM build UPX cd %BDIR%\upx set s=%H%\upx\src cat .GITREV.txt set /p GITREV=<.GITREV.txt - cl -MT -EHsc -J -O2 -W4 -WX -DUPX_VERSION_GITREV="""%GITREV%""" %DEFS% -I%s%\lzma-sdk -I%H%\deps\ucl-1.03\include -I%H%\deps\zlib-1.2.11 -Feupx.exe %s%\*.cpp %BDIR%\ucl-1.03\ucl.lib %BDIR%\zlib-1.2.11\zlib.lib /link setargv.obj + cl -MT -EHsc -J -O2 -W4 -WX -DUPX_VERSION_GITREV="""%GITREV%""" %DEFS% -I%H%\upx\vendor\doctest -I%H%\upx\vendor\lzma-sdk -I%H%\upx\vendor\ucl\include -I%H%\upx\vendor\zlib -Feupx.exe %s%\*.cpp %BDIR%\ucl\ucl.lib %BDIR%\zlib\zlib.lib /link setargv.obj - name: 'Basic tests' shell: cmd @@ -245,15 +342,19 @@ jobs: bash ./.github/travis_testsuite_1.sh - name: 'Make artifact' - shell: cmd + id: make_artifact + shell: bash run: | - mkdir tmp - copy %H%\build\%C%\%B%\upx\upx_packed.exe tmp\upx.exe + N=upx-${GITHUB_REF_NAME}-${GITHUB_SHA:0:7}-${MATRIX_NAME} + mkdir -p "tmp/artifact/$N/$B" + cp -ai ../build/$C/$B/upx/upx*.exe "tmp/artifact/$N/$B" + # GitHub Actions magic: set "upload_name" Javascript outputs value for use in next step + echo "::set-output name=upload_name::$N" - name: 'Upload artifact' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: - name: ${{ matrix.name }} - path: tmp + name: ${{ steps.make_artifact.outputs.upload_name }} + path: tmp/artifact # vim:set ts=2 sw=2 et: diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml index ac88f228..a44647f8 100644 --- a/.github/workflows/close-stale-issues.yml +++ b/.github/workflows/close-stale-issues.yml @@ -1,13 +1,23 @@ # see https://docs.github.com/en/actions/managing-issues-and-pull-requests/closing-inactive-issues # see https://github.com/actions/stale -name: Close inactive issues +# Automatically closing issues is far from perfect, but then we only have +# limited resources and this approach favors issues that people +# actually care about. +# +# also see: +# "Should I Stale or Should I Close? An Analysis of a Bot that Closes +# Abandoned Issues and Pull Request". +# https://igorwiese.com/images/papers/Paper_BotSE_19.pdf + +name: 'Close inactive issues' on: schedule: - cron: "30 6 * * *" jobs: close-issues: + name: 'Close inactive issues' runs-on: ubuntu-latest permissions: issues: write diff --git a/.github/workflows/nopr.yml b/.github/workflows/nopr.yml new file mode 100644 index 00000000..eb4ab7f8 --- /dev/null +++ b/.github/workflows/nopr.yml @@ -0,0 +1,16 @@ +name: 'Close pull requests' +on: + #pull_request: + workflow_dispatch: +jobs: + close-pull-requests: + name: 'Close pull requests' + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: 'Close pull requests' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr list --repo='${{ github.repository }}' --state=open --json=number | jq '.[].number' | xargs -r -n1 -- gh pr close --repo='${{ github.repository }}' diff --git a/src/stub/scripts/upx-clang-format b/src/stub/scripts/upx-clang-format index fee5628a..5f925d0b 100755 --- a/src/stub/scripts/upx-clang-format +++ b/src/stub/scripts/upx-clang-format @@ -7,14 +7,16 @@ set -e; set -o pipefail # NOTE: we are using clang-format-10.0.1 from upx-stubtools # see https://github.com/upx/upx-stubtools/releases -CLANG_FORMAT="$HOME/local/bin/bin-upx/clang-format-10.0.1" -if [[ ! -f $CLANG_FORMAT ]]; then -CLANG_FORMAT="$HOME/.local/bin/bin-upx/clang-format-10.0.1" +if [[ ! -f $UPX_CLANG_FORMAT ]]; then +UPX_CLANG_FORMAT="$HOME/local/bin/bin-upx/clang-format-10.0.1" fi -if [[ ! -f $CLANG_FORMAT ]]; then -CLANG_FORMAT="$HOME/bin/bin-upx/clang-format-10.0.1" +if [[ ! -f $UPX_CLANG_FORMAT ]]; then +UPX_CLANG_FORMAT="$HOME/.local/bin/bin-upx/clang-format-10.0.1" fi -if [[ ! -f $CLANG_FORMAT ]]; then +if [[ ! -f $UPX_CLANG_FORMAT ]]; then +UPX_CLANG_FORMAT="$HOME/bin/bin-upx/clang-format-10.0.1" +fi +if [[ ! -f $UPX_CLANG_FORMAT ]]; then echo "ERROR: $0: cannot find clang-format-10.0.1" echo "ERROR: $0: please visit https://github.com/upx/upx-stubtools" exit 1 @@ -22,6 +24,6 @@ fi # NOTE: we use .clang-format config from upx.git/.clang-format -#echo $CLANG_FORMAT -exec "$CLANG_FORMAT" -style=file "$@" +#echo $UPX_CLANG_FORMAT +exec "$UPX_CLANG_FORMAT" -style=file "$@" exit 1