src: fix m68k-atari build, prepare for std::atomic, port John's MemBuffer debug

src/conf.h

@@ -117,6 +117,9 @@ ACC_COMPILE_TIME_ASSERT_HEADER((char)(-1) == 255) // -funsigned-char
 #define ACC_WANT_ACC_LIB_H 1
 #define ACC_WANT_ACC_CXX_H 1
 #include "miniacc.h"
+#if (ACC_CC_MSC)
+#  include <intrin.h>
+#endif
 
 /* intergral types */
 typedef acc_int8_t      upx_int8_t;
@@ -264,12 +267,14 @@ typedef upx_int64_t upx_off_t;
 #endif
 
 // avoid warnings about shadowing global functions
+#undef _base
 #undef basename
 #undef index
 #undef outp
-#define basename            upx_basename
+#define _base               upx_renamed__base
-#define index               upx_index
+#define basename            upx_renamed_basename
-#define outp                upx_outp
+#define index               upx_renamed_index
+#define outp                upx_renamed_outp
 
 #undef PAGE_MASK
 #undef PAGE_SIZE
@@ -321,6 +326,16 @@ inline void NO_fprintf(FILE *, const char *, ...) {}
 #  define upx_memcpy_inline     memcpy
 #endif
 
+#if __has_builtin(__builtin_return_address)
+#  define upx_return_address()  __builtin_return_address(0)
+#elif defined(__GNUC__)
+#  define upx_return_address()  __builtin_return_address(0)
+#elif (ACC_CC_MSC)
+#  define upx_return_address()  _ReturnAddress()
+#else
+#  define upx_return_address()  nullptr
+#endif
+
 #define UNUSED(var)             ACC_UNUSED(var)
 #define COMPILE_TIME_ASSERT(e)  ACC_COMPILE_TIME_ASSERT(e)
 
@@ -328,6 +343,14 @@ inline void NO_fprintf(FILE *, const char *, ...) {}
 #define __packed_struct(s)      struct alignas(1) s {
 #define __packed_struct_end()   };
 
+#if (ACC_ARCH_M68K && ACC_OS_TOS && ACC_CC_GNUC) && defined(__MINT__)
+// hack for broken compiler
+#define upx_alignas_1       __attribute__((__aligned__(1),__packed__))
+#define upx_alignas_16      __attribute__((__aligned__(2))) // object file maximum 2 ???
+#define upx_alignas__(a)    upx_alignas_ ## a
+#define alignas(x)          upx_alignas__(x)
+#endif
+
 #define COMPILE_TIME_ASSERT_ALIGNOF_USING_SIZEOF__(a,b) { \
      typedef a acc_tmp_a_t; typedef b acc_tmp_b_t; \
      struct alignas(1) acc_tmp_t { acc_tmp_b_t x; acc_tmp_a_t y; acc_tmp_b_t z; }; \
@@ -730,6 +753,14 @@ struct upx_compress_result_t
 #include <new>
 #include <type_traits>
 #include <typeinfo>
+#if __STDC_NO_ATOMICS__ || 1
+// UPX currently does not use multithreading
+#define upx_std_atomic(Type)    Type
+//#define upx_std_atomic(Type)    typename std::add_volatile<Type>::type
+#else
+#include <atomic>
+#define upx_std_atomic(Type)    std::atomic<Type>
+#endif
 
 #include "options.h"
 #include "except.h"

src/console.h

@@ -62,6 +62,8 @@
 #elif 1 && (ACC_OS_EMX && defined(__RSXNT__))
 #define USE_SCREEN 1
 #define USE_SCREEN_WIN32 1
+#elif 1 && (ACC_ARCH_M68K && ACC_OS_TOS)
+#define NO_CONSOLE 1
 #endif
 
 #if 0 || (NO_ANSI)

src/util/dt_impl.cpp

@@ -33,8 +33,12 @@
 #if !defined(UPX_DOCTEST_CONFIG_MULTITHREADING)
 #define DOCTEST_CONFIG_NO_MULTITHREADING
 #endif
-#if defined(__MSDOS__) && defined(__DJGPP__)
+#if defined(__i386__) && defined(__MSDOS__) && defined(__DJGPP__) && defined(__GNUC__)
 #define DOCTEST_CONFIG_NO_POSIX_SIGNALS
+#elif defined(__m68k__) && defined(__atarist__) && defined(__GNUC__)
+#define DOCTEST_CONFIG_COLORS_NONE
+#define DOCTEST_CONFIG_NO_POSIX_SIGNALS
+#pragma GCC diagnostic ignored "-Wshadow"
 #endif
 #define DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
 #if !defined(DOCTEST_CONFIG_DISABLE)

src/util/membuffer.cpp

@@ -32,6 +32,14 @@
 void *membuffer_get_void_ptr(MemBuffer &mb) { return mb.getVoidPtr(); }
 unsigned membuffer_get_size(MemBuffer &mb) { return mb.getSize(); }
 
+MemBuffer::Stats MemBuffer::stats;
+
+#if DEBUG
+#define debug_set(var, expr) (var) = (expr)
+#else
+#define debug_set(var, expr) /*empty*/
+#endif
+
 /*************************************************************************
 // bool use_simple_mcheck()
 **************************************************************************/
@@ -60,13 +68,17 @@ __acc_static_forceinline constexpr bool use_simple_mcheck() { return true; }
 //
 **************************************************************************/
 
-MemBuffer::MemBuffer(upx_uint64_t size_in_bytes) { alloc(size_in_bytes); }
+MemBuffer::MemBuffer(upx_uint64_t size_in_bytes) {
+    alloc(size_in_bytes);
+    debug_set(debug.last_return_address_alloc, upx_return_address());
+}
 
 MemBuffer::~MemBuffer() { this->dealloc(); }
 
 // similar to BoundedPtr, except checks only at creation
 // skip == offset, take == size_in_bytes
 void *MemBuffer::subref_impl(const char *errfmt, size_t skip, size_t take) {
+    debug_set(debug.last_return_address_subref, upx_return_address());
     // check overrun and wrap-around
     if (skip + take > b_size_in_bytes || skip + take < skip) {
         char buf[100];
@@ -130,14 +142,17 @@ unsigned MemBuffer::getSizeForDecompression(unsigned uncompressed_size, unsigned
 void MemBuffer::allocForCompression(unsigned uncompressed_size, unsigned extra) {
     unsigned size = getSizeForCompression(uncompressed_size, extra);
     alloc(size);
+    debug_set(debug.last_return_address_alloc, upx_return_address());
 }
 
 void MemBuffer::allocForDecompression(unsigned uncompressed_size, unsigned extra) {
     unsigned size = getSizeForDecompression(uncompressed_size, extra);
     alloc(size);
+    debug_set(debug.last_return_address_alloc, upx_return_address());
 }
 
 void MemBuffer::fill(unsigned off, unsigned len, int value) {
+    debug_set(debug.last_return_address_fill, upx_return_address());
     checkState();
     assert((int) off >= 0);
     assert((int) len >= 0);
@@ -156,8 +171,6 @@ void MemBuffer::fill(unsigned off, unsigned len, int value) {
 #define MAGIC1(p) ((PTR_BITS(p) ^ 0xfefdbeeb) | 1)
 #define MAGIC2(p) ((PTR_BITS(p) ^ 0xfefdbeeb ^ 0x80024011) | 1)
 
-unsigned MemBuffer::global_alloc_counter = 0;
-
 void MemBuffer::checkState() const {
     if (!b)
         throwInternalError("block not allocated");
@@ -177,8 +190,10 @@ void MemBuffer::alloc(upx_uint64_t size) {
     assert(b_size_in_bytes == 0);
     //
     assert(size > 0);
+    debug_set(debug.last_return_address_alloc, upx_return_address());
     size_t bytes = mem_size(1, size, use_simple_mcheck() ? 32 : 0);
     unsigned char *p = (unsigned char *) malloc(bytes);
+    NO_printf("MemBuffer::alloc %llu: %p\n", size, p);
     if (!p)
         throwOutOfMemoryException();
     b = p;
@@ -189,17 +204,22 @@ void MemBuffer::alloc(upx_uint64_t size) {
         set_ne32(b - 8, b_size_in_bytes);
         set_ne32(b - 4, MAGIC1(b));
         set_ne32(b + b_size_in_bytes, MAGIC2(b));
-        set_ne32(b + b_size_in_bytes + 4, global_alloc_counter++);
+        set_ne32(b + b_size_in_bytes + 4, stats.global_alloc_counter);
     }
 #if !defined(__SANITIZE_ADDRESS__) && 0
     fill(0, b_size_in_bytes, (rand() & 0xff) | 1); // debug
     (void) VALGRIND_MAKE_MEM_UNDEFINED(b, b_size_in_bytes);
 #endif
+    stats.global_alloc_counter += 1;
+    stats.global_total_bytes += b_size_in_bytes;
+    stats.global_total_active_bytes += b_size_in_bytes;
 }
 
 void MemBuffer::dealloc() {
     if (b != nullptr) {
+        debug_set(debug.last_return_address_dealloc, upx_return_address());
         checkState();
+        stats.global_total_active_bytes -= b_size_in_bytes;
         if (use_simple_mcheck()) {
             // clear magic constants
             set_ne32(b - 8, 0);

src/util/membuffer.h

@@ -71,7 +71,7 @@ public:
     }
 };
 
-class MemBuffer : public MemBufferBase<unsigned char> {
+class MemBuffer final : public MemBufferBase<unsigned char> {
 public:
     MemBuffer() = default;
     explicit MemBuffer(upx_uint64_t size_in_bytes);
@@ -94,21 +94,37 @@ public:
 
     // util
     void fill(unsigned off, unsigned len, int value);
-    void clear(unsigned off, unsigned len) { fill(off, len, 0); }
+    __acc_forceinline void clear(unsigned off, unsigned len) { fill(off, len, 0); }
-    void clear() { fill(0, b_size_in_bytes, 0); }
+    __acc_forceinline void clear() { fill(0, b_size_in_bytes, 0); }
 
     // If the entire range [skip, skip+take) is inside the buffer,
     // then return &b[skip]; else throwCantPack(sprintf(errfmt, skip, take)).
     // This is similar to BoundedPtr, except only checks once.
     // skip == offset, take == size_in_bytes
-    pointer subref(const char *errfmt, size_t skip, size_t take) {
+    __acc_forceinline pointer subref(const char *errfmt, size_t skip, size_t take) {
         return (pointer) subref_impl(errfmt, skip, take);
     }
 
 private:
     void *subref_impl(const char *errfmt, size_t skip, size_t take);
 
-    static unsigned global_alloc_counter;
+    // static debug stats
+    struct Stats {
+        upx_std_atomic(upx_uint32_t) global_alloc_counter;
+        upx_std_atomic(upx_uint64_t) global_total_bytes;
+        upx_std_atomic(upx_uint64_t) global_total_active_bytes;
+    };
+    static Stats stats;
+#if DEBUG
+    // debugging aid
+    struct Debug {
+        void *last_return_address_alloc = nullptr;
+        void *last_return_address_dealloc = nullptr;
+        void *last_return_address_fill = nullptr;
+        void *last_return_address_subref = nullptr;
+    };
+    Debug debug;
+#endif
 
     // disable copy, assignment and move assignment
     MemBuffer(const MemBuffer &) = delete;

src/util/xspan.cpp

@@ -30,7 +30,7 @@
 
 SPAN_NAMESPACE_BEGIN
 
-unsigned long long span_check_stats_check_range = 0;
+upx_std_atomic(upx_uint64_t) span_check_stats_check_range_counter(0);
 
 // HINT: set env-var "UPX_DEBUG_DOCTEST_DISABLE=1" for improved debugging experience
 __acc_noinline void span_fail_nullptr() {
@@ -61,7 +61,7 @@ void span_check_range(const void *p, const void *base, ptrdiff_t size_in_bytes)
     ptrdiff_t off = (const char *) p - (const char *) base;
     if __acc_very_unlikely (off < 0 || off > size_in_bytes)
         span_fail_range_range();
-    span_check_stats_check_range += 1;
+    span_check_stats_check_range_counter += 1;
     // fprintf(stderr, "span_check_range done\n");
 }