CI updates

This commit is contained in:
Markus F.X.J. Oberhumer 2023-09-11 06:52:57 +02:00
parent 0192b0b7e4
commit 57ad6bc37d
12 changed files with 94 additions and 67 deletions

View File

@ -12,8 +12,8 @@ env:
CMAKE_REQUIRED_QUIET: OFF
DEBIAN_FRONTEND: noninteractive
UPX_CMAKE_BUILD_FLAGS: --verbose
# 2023-09-05
ZIG_DIST_VERSION: 0.12.0-dev.280+64d03faae
# 2023-09-10
ZIG_DIST_VERSION: 0.12.0-dev.294+4d1432299
jobs:
job-rebuild-and-verify-stubs:

View File

@ -14,7 +14,7 @@ jobs:
strategy:
fail-fast: false
matrix: {container: ['alpine:3.16','alpine:3.17','alpine:3.18','alpine:edge','i386/alpine:edge']}
name: ${{ format('Analyze {0}', matrix.container) }}
name: ${{ format('Analyze clang-analyzer {0}', matrix.container) }}
runs-on: ubuntu-latest
container: ${{ matrix.container }}
steps:

View File

@ -8,13 +8,14 @@ on:
env:
CMAKE_REQUIRED_QUIET: OFF
DEBIAN_FRONTEND: noninteractive
VERBOSE: 1
jobs:
job-alpine-by-hand: # uses a POSIX-compliant shell
# ...and also uses a subdirectory "upx with space" in order to detect possible quoting issues
# ...and also uses ccache as we are running the same build-script again and again
if: github.repository_owner == 'upx'
strategy: { matrix: { container: ['alpine:3.9','alpine:3.18','alpine:edge'] } }
strategy: { matrix: { container: ['alpine:3.9','alpine:3.18','alpine:edge','i386/alpine:edge'] } }
name: ${{ format('gcc by-hand {0}', matrix.container) }}
runs-on: ubuntu-latest
container: ${{ matrix.container }}
@ -28,15 +29,17 @@ jobs:
esac
echo "installing shells: $shells"
apk update && apk upgrade && apk add ccache g++ git $shells
# enable ccache
echo -e "CC=ccache gcc\nCXX=ccache g++ -std=gnu++17" >> $GITHUB_ENV
# enable ccache and some warnings
warn="-Wall -Wextra -Werror"
echo -e "CC=ccache gcc $warn\nCXX=ccache g++ -std=gnu++17 $warn" >> $GITHUB_ENV
# this seems to be needed when running in a container (beause of UID mismatch??)
git config --global --add safe.directory '*'
- name: 'Check out code'
uses: actions/checkout@v4
with:
submodules: true
path: 'upx with space'
# create user upx:upx 2000:2000 for file system tests below ("sudo")
adduser upx -u 2000 -D && cd /home/upx && chmod 00700 . && chown -R upx:upx .
- name: ${{ format('Check out UPX {0} source code', github.ref_name) }}
run: |
git clone --branch "$GITHUB_REF_NAME" --depth 1 https://github.com/upx/upx "upx with space"
git -C "upx with space" submodule update --init
- name: 'Build by-hand with bash'
run: 'bash "./upx with space/misc/scripts/build_upx_by_hand.sh"'
- name: 'Build by-hand with bash --posix'
@ -81,3 +84,17 @@ jobs:
run: |
ccache -s
ccache -p
- name: 'Run file system test suite (busybox)'
run: |
apk add bash sudo
testsuite="$(readlink -fn "upx with space"/misc/testsuite/test_symlinks.sh)"
cd "upx with space"/build/by-hand
# IMPORTANT: do NOT run as user root!
chmod a+w . && sudo -u upx bash "$testsuite"
- name: 'Run file system test suite (coreutils)'
run: |
apk add bash coreutils sudo
testsuite="$(readlink -fn "upx with space"/misc/testsuite/test_symlinks.sh)"
cd "upx with space"/build/by-hand
# IMPORTANT: do NOT run as user root!
chmod a+w . && sudo -u upx bash "$testsuite"

View File

@ -37,6 +37,8 @@ jobs:
# clang-dev is needed on older Alpine versions for clang headers like <emmintrin.h>
*:3.[0-9]|*:3.10|*:3.11) apk add clang-dev ;;
esac
# create user upx:upx 2000:2000 for file system tests below ("sudo")
adduser upx -u 2000 -D && cd /home/upx && chmod 00700 . && chown -R upx:upx .
- name: ${{ format('Check out UPX {0} source code', github.ref_name) }}
run: |
git clone --branch "$GITHUB_REF_NAME" --depth 1 https://github.com/upx/upx "upx with space"
@ -63,55 +65,55 @@ jobs:
# build with C17 and C++20 on alpine:edge
- name: ${{ format('Build clang C++20 Release with {0}', env.clang_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=clang-cxx20-static CC="clang -std=gnu17 -static" CXX="clang++ -std=gnu++20 -static"
- name: ${{ format('Build clang C++20 Debug with {0}', env.clang_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=clang-cxx20-static CC="clang -std=gnu17 -static" CXX="clang++ -std=gnu++20 -static" xtarget/debug
- name: ${{ format('Build gcc C++20 Release with {0}', env.gcc_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=gcc-cxx20-static CC="gcc -std=gnu17 -static" CXX="g++ -std=gnu++20 -static"
- name: ${{ format('Build gcc C++20 Debug with {0}', env.gcc_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=gcc-cxx20-static CC="gcc -std=gnu17 -static" CXX="g++ -std=gnu++20 -static" xtarget/debug
# build with C23 and C++23 on alpine:edge
- name: ${{ format('Build clang C++23 Release with {0}', env.clang_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=clang-cxx23-static CC="clang -std=gnu2x -static" CXX="clang++ -std=gnu++2b -static"
- name: ${{ format('Build clang C++23 Debug with {0}', env.clang_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=clang-cxx23-static CC="clang -std=gnu2x -static" CXX="clang++ -std=gnu++2b -static" xtarget/debug
- name: ${{ format('Build gcc C++23 Release with {0}', env.gcc_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=gcc-cxx23-static CC="gcc -std=gnu2x -static" CXX="g++ -std=gnu++23 -static"
- name: ${{ format('Build gcc C++23 Debug with {0}', env.gcc_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=gcc-cxx23-static CC="gcc -std=gnu2x -static" CXX="g++ -std=gnu++23 -static" xtarget/debug
# build with -flto=auto on alpine:edge
- name: ${{ format('Build clang LTO Release with {0}', env.clang_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=clang-cxxlto-static CC="clang -flto=auto -static" CXX="clang++ -flto=auto -static"
- name: ${{ format('Build clang LTO Debug with {0}', env.clang_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=clang-cxxlto-static CC="clang -flto=auto -static" CXX="clang++ -flto=auto -static" xtarget/debug
- name: ${{ format('Build gcc LTO Release with {0}', env.gcc_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=gcc-cxxlto-static CC="gcc -flto=auto -static" CXX="g++ -flto=auto -static"
- name: ${{ format('Build gcc LTO Debug with {0}', env.gcc_package) }}
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
make -C "upx with space" UPX_XTARGET=gcc-cxxlto-static CC="gcc -flto=auto -static" CXX="g++ -flto=auto -static" xtarget/debug
@ -129,7 +131,7 @@ jobs:
- { name: 'Run basic tests gcc Release', run: 'make -C "upx with space"/build/xtarget/gcc-static/release test' }
- { name: 'Run basic tests gcc Debug', run: 'make -C "upx with space"/build/xtarget/gcc-static/debug test' }
- name: 'Run basic tests C++20, C++23 and LTO'
if: ${{ contains(matrix.container, ':edge') }}
if: endsWith(matrix.container, ':edge')
run: |
for dir in "upx with space"/build/xtarget/*-cxx*/*; do
echo "===== $dir"
@ -152,7 +154,7 @@ jobs:
testsuite="$(readlink -fn "upx with space"/misc/testsuite/test_symlinks.sh)"
cd "upx with space"/build/xtarget/gcc-static/release
# IMPORTANT: do NOT run as user root!
chmod a+w . && sudo -u operator bash "$testsuite"
chmod a+w . && sudo -u upx bash "$testsuite"
# test suite
- name: ${{ format('Run test suite level {0}', env.UPX_TESTSUITE_LEVEL) }}
@ -171,4 +173,4 @@ jobs:
testsuite="$(readlink -fn "upx with space"/misc/testsuite/test_symlinks.sh)"
cd "upx with space"/build/xtarget/gcc-static/release
# IMPORTANT: do NOT run as user root!
chmod a+w . && sudo -u operator bash "$testsuite"
chmod a+w . && sudo -u upx bash "$testsuite"

View File

@ -24,12 +24,12 @@ jobs:
- name: llvm-mingw-20230614-ucrt
llvm_version: 16.0.6
url: 'https://github.com/mstorsjo/llvm-mingw/releases/download/20230614/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64.tar.xz'
- name: llvm-mingw-20230822-msvcrt
llvm_version: 17.0.0rc3
url: 'https://github.com/mstorsjo/llvm-mingw/releases/download/20230822/llvm-mingw-20230822-msvcrt-ubuntu-20.04-x86_64.tar.xz'
- name: llvm-mingw-20230822-ucrt
llvm_version: 17.0.0rc3
url: 'https://github.com/mstorsjo/llvm-mingw/releases/download/20230822/llvm-mingw-20230822-ucrt-ubuntu-20.04-x86_64.tar.xz'
- name: llvm-mingw-20230905-msvcrt
llvm_version: 17.0.0rc4
url: 'https://github.com/mstorsjo/llvm-mingw/releases/download/20230905/llvm-mingw-20230905-msvcrt-ubuntu-20.04-x86_64.tar.xz'
- name: llvm-mingw-20230905-ucrt
llvm_version: 17.0.0rc4
url: 'https://github.com/mstorsjo/llvm-mingw/releases/download/20230905/llvm-mingw-20230905-ucrt-ubuntu-20.04-x86_64.tar.xz'
name: ${{ format('{0} {1}', matrix.name, matrix.llvm_version) }}
runs-on: ubuntu-latest
steps:

View File

@ -10,8 +10,8 @@ on:
env:
CMAKE_REQUIRED_QUIET: OFF
DEBIAN_FRONTEND: noninteractive
# 2023-09-05
ZIG_DIST_VERSION: 0.12.0-dev.280+64d03faae
# 2023-09-10
ZIG_DIST_VERSION: 0.12.0-dev.294+4d1432299
jobs:
job-linux-zigcc: # uses cmake + make

View File

@ -99,5 +99,5 @@ jobs:
export upx_exe_runner="valgrind --error-exitcode=1 --quiet"
# on current GitHub CI, takes about 30 minutes for release and 80 minutes for debug builds
# reduce time for debug builds to about 30 minutes
test $release = debug && export UPX_TESTSUITE_LEVEL=4
test "$release" = "debug" && export UPX_TESTSUITE_LEVEL=4
env -C build/xtarget/clang-static/$release "$PWD"/misc/testsuite/upx_testsuite_1.sh

View File

@ -8,7 +8,7 @@ set -e
# Copyright (C) Markus Franz Xaver Johannes Oberhumer
#
# uses optional environment variables: AR, CC, CXX, OPTIMIZE, top_srcdir
# uses optional environment variables: AR, CC, CXX, OPTIMIZE, VERBOSE, top_srcdir
# shell init
### set -x # enable logging
@ -62,7 +62,12 @@ run() {
fi
# print short info and run command
test "x$1" != "x" && test "x$1" != "x+" && echo "$1"
shift; "$@"
shift
if test "x$VERBOSE" != "x" && test "x$VERBOSE" != "x0"; then
# print full command
echo " $@"
fi
"$@"
}
# helper function

View File

@ -8,12 +8,13 @@ argv0=$0; argv0abs=$(readlink -fn "$argv0"); argv0dir=$(dirname "$argv0abs")
umask 0022
id || true
echo "PWD='$PWD'"
if [[ $UID == 0 || $EUID == 0 ]]; then
echo "ERROR: do not run as root: UID=$UID EUID=$EUID"
exit 91
fi
# test behaviour with symlinks; requires:
# test file system behaviour with symlinks; requires:
# $upx_exe (required, but with convenience fallback "./upx")
# optional settings:
# $upx_exe_runner (e.g. "qemu-x86_64 -cpu Westmere" or "valgrind")
@ -39,15 +40,19 @@ fi
upx_run+=( "$upx_exe" )
echo "upx_run='${upx_run[*]}'"
# upx_run check, part1
# upx_run check
if ! "${upx_run[@]}" --version-short >/dev/null; then echo "UPX-ERROR: FATAL: upx --version-short FAILED"; exit 1; fi
if ! "${upx_run[@]}" -L >/dev/null 2>&1; then echo "UPX-ERROR: FATAL: upx -L FAILED"; exit 1; fi
if ! "${upx_run[@]}" --help >/dev/null; then echo "UPX-ERROR: FATAL: upx --help FAILED"; exit 1; fi
#***********************************************************************
#
# util
#***********************************************************************
exit_code=0
num_errors=0
all_errors=
failed() {
####exit $1
# log error and keep going
@ -97,6 +102,7 @@ assert_symlink_dangling() {
create_files() {
# clean
local d
for d in z_dir_1 z_dir_2 z_dir_3 z_dir_4; do
if [[ -d $d ]]; then
chmod -R +w "./$d"
@ -137,16 +143,13 @@ create_files() {
#
#***********************************************************************
#set -x # debug
export UPX="--prefer-ucl --no-color --no-progress"
export UPX_DEBUG_DISABLE_GITREV_WARNING=1
export UPX_DEBUG_DOCTEST_VERBOSE=0
export NO_COLOR=1
#set -x # debug
exit_code=0
num_errors=0
all_errors=
testsuite_header() {
local x='==========='; x="$x$x$x$x$x$x$x"
echo -e "\n${x}\n${1}\n${x}\n"

View File

@ -71,9 +71,8 @@
FileBase::~FileBase() may_throw {
#if 0 && defined(__GNUC__) // debug
if (isOpen())
fprintf(stderr,"%s: %s\n", _name, __PRETTY_FUNCTION__);
fprintf(stderr, "%s: %s\n", _name, __PRETTY_FUNCTION__);
#endif
if (std::uncaught_exceptions() == 0)
closex(); // may_throw
else

View File

@ -49,7 +49,7 @@ ACC_COMPILE_TIME_ASSERT_HEADER(UPX_RSIZE_MAX_MEM == UPX_RSIZE_MAX)
ACC_COMPILE_TIME_ASSERT_HEADER(UPX_RSIZE_MAX_STR <= UPX_RSIZE_MAX / 256)
ACC_COMPILE_TIME_ASSERT_HEADER(2ull * UPX_RSIZE_MAX * 9 / 8 + 256 * 1024 * 1024 < INT_MAX)
ACC_COMPILE_TIME_ASSERT_HEADER(2ull * UPX_RSIZE_MAX * 10 / 8 + 128 * 1024 * 1024 <= INT_MAX + 1u)
ACC_COMPILE_TIME_ASSERT_HEADER(5ull * UPX_RSIZE_MAX < UINT_MAX)
ACC_COMPILE_TIME_ASSERT_HEADER(5ull * UPX_RSIZE_MAX < UINT_MAX) // IMPORTANT overflow protection
ACC_COMPILE_TIME_ASSERT_HEADER(UPX_RSIZE_MAX >= 8192 * 65536)
ACC_COMPILE_TIME_ASSERT_HEADER(UPX_RSIZE_MAX_STR >= 1024)
@ -264,12 +264,12 @@ void *upx_calloc(size_t n, size_t element_size) {
// simple unoptimized memswap()
void upx_memswap(void *a, void *b, size_t n) {
if (a != b && n != 0) {
char *x = (char *) a;
char *y = (char *) b;
byte *x = (byte *) a;
byte *y = (byte *) b;
do {
// strange clang-analyzer-15 false positive when compiling in Debug mode
// clang-analyzer-core.uninitialized.Assign
char tmp = *x; // NOLINT(*core.uninitialized.Assign) // bogus clang-analyzer warning
byte tmp = *x; // NOLINT(*core.uninitialized.Assign) // bogus clang-analyzer warning
*x++ = *y;
*y++ = tmp;
} while (--n != 0);
@ -277,12 +277,12 @@ void upx_memswap(void *a, void *b, size_t n) {
}
// much better memswap(), optimized for our use case in sort functions below
static void memswap_no_overlap(char *a, char *b, size_t n) {
static void memswap_no_overlap(byte *a, byte *b, size_t n) {
#if defined(__clang__) && __clang_major__ < 15
// work around a clang < 15 ICE (Internal Compiler Error)
upx_memswap(a, b, n);
#else // clang bug
upx_alignas_max char tmp_buf[16];
upx_alignas_max byte tmp_buf[16];
#define SWAP(x) \
ACC_BLOCK_BEGIN \
upx_memcpy_inline(tmp_buf, a, x); \
@ -301,7 +301,7 @@ static void memswap_no_overlap(char *a, char *b, size_t n) {
if (n & 2)
SWAP(2);
if (n & 1) {
char tmp = *a;
byte tmp = *a;
*a = *b;
*b = tmp;
}
@ -313,7 +313,7 @@ static void memswap_no_overlap(char *a, char *b, size_t n) {
// WARNING: O(n^2) and thus very inefficient for large n
void upx_gnomesort(void *array, size_t n, size_t element_size, upx_compare_func_t compare) {
for (size_t i = 1; i < n; i++) {
char *a = (char *) array + element_size * i; // a := &array[i]
byte *a = (byte *) array + element_size * i; // a := &array[i]
if (i != 0 && compare(a - element_size, a) > 0) { // if a[-1] > a[0] then
memswap_no_overlap(a - element_size, a, element_size); // swap elements a[-1] <=> a[0]
i -= 2; // and decrease i
@ -330,22 +330,22 @@ void upx_shellsort_memswap(void *array, size_t n, size_t element_size, upx_compa
gap = gap * 3 + 1;
for (; gap > 0; gap = (gap - 1) / 3) {
const size_t gap_bytes = element_size * gap;
char *p = (char *) array + gap_bytes;
byte *p = (byte *) array + gap_bytes;
for (size_t i = gap; i < n; i += gap, p += gap_bytes) // invariant: p == &array[i]
for (char *a = p; a != array && compare(a - gap_bytes, a) > 0; a -= gap_bytes)
for (byte *a = p; a != array && compare(a - gap_bytes, a) > 0; a -= gap_bytes)
memswap_no_overlap(a - gap_bytes, a, element_size);
}
}
// simple Shell sort using Knuth's gap; NOT stable; uses memcpy()
// should be faster than memswap() in theory, but benchmarks are inconsistent
// should be faster than memswap() version in theory, but benchmarks are inconsistent
void upx_shellsort_memcpy(void *array, size_t n, size_t element_size, upx_compare_func_t compare) {
mem_size_assert(element_size, n); // check size
constexpr size_t MAX_INLINE_ELEMENT_SIZE = 256;
upx_alignas_max char tmp_buf[MAX_INLINE_ELEMENT_SIZE]; // buffer for one element
char *tmp = tmp_buf;
upx_alignas_max byte tmp_buf[MAX_INLINE_ELEMENT_SIZE]; // buffer for one element
byte *tmp = tmp_buf;
if (element_size > MAX_INLINE_ELEMENT_SIZE) {
tmp = (char *) malloc(element_size);
tmp = (byte *) malloc(element_size);
assert(tmp != nullptr);
}
size_t gap = 0; // 0, 1, 4, 13, 40, 121, 364, 1093, ...
@ -353,10 +353,10 @@ void upx_shellsort_memcpy(void *array, size_t n, size_t element_size, upx_compar
gap = gap * 3 + 1;
for (; gap > 0; gap = (gap - 1) / 3) {
const size_t gap_bytes = element_size * gap;
char *p = (char *) array + gap_bytes;
byte *p = (byte *) array + gap_bytes;
for (size_t i = gap; i < n; i += gap, p += gap_bytes) // invariant: p == &array[i]
if (compare(p - gap_bytes, p) > 0) {
char *a = p;
byte *a = p;
memcpy(tmp, a, element_size);
do {
memcpy(a, a - gap_bytes, element_size);
@ -378,7 +378,7 @@ void upx_std_stable_sort(void *array, size_t n, upx_compare_func_t compare) {
// just for testing
upx_gnomesort(array, n, ElementSize, compare);
#else
struct alignas(1) element_type { char data[ElementSize]; };
struct alignas(1) element_type { byte data[ElementSize]; };
static_assert(sizeof(element_type) == ElementSize);
static_assert(alignof(element_type) == 1);
auto cmp = [compare](const element_type &a, const element_type &b) -> bool {

View File

@ -69,7 +69,7 @@ static constexpr int get_open_flags(OpenMode om) noexcept {
if (om == WO_CREATE_OR_TRUNCATE)
return wo_flags | O_CREAT | O_TRUNC; // create if not exists, otherwise truncate
// RO_MUST_EXIST
return O_RDONLY | O_BINARY;
return O_RDONLY | O_BINARY; // will cause an error if file does not exist
}
static void copy_file_contents(const char *iname, const char *oname, OpenMode om) may_throw {
@ -127,6 +127,7 @@ static void copy_file_attributes(const struct stat *st, const char *oname, bool
}
#endif
// maybe unused
UNUSED(st);
UNUSED(oname);
UNUSED(preserve_mode);
UNUSED(preserve_ownership);
@ -221,7 +222,7 @@ void do_one_file(const char *const iname, char *const oname) may_throw {
if (!maketempname(tname, sizeof(tname), iname, ".upx"))
throwIOException("could not create a temporary file name");
}
int flags = get_open_flags(WO_MUST_CREATE);
int flags = get_open_flags(WO_MUST_CREATE); // don't overwrite files by default
if (opt->output_name && preserve_link) {
flags = get_open_flags(WO_CREATE_OR_TRUNCATE);
#if HAVE_LSTAT
@ -289,8 +290,8 @@ void do_one_file(const char *const iname, char *const oname) may_throw {
}
// close files
fo.closex();
fi.closex();
fo.closex();
// rename or copy files
if (oname[0] && !opt->output_name) {