Stub msync() on PF_X before munmap()

modified:   stub/src/amd64-linux.elf-fold.S
	modified:   stub/src/amd64-linux.elf-main2.c
	modified:   stub/src/arm.v4a-linux.elf-fold.S
	modified:   stub/src/arm64-linux.elf-entry.S
	modified:   stub/src/arm64-linux.elf-fold.S
	modified:   stub/src/i386-linux.elf-main2.c
	modified:   stub/src/include/linux.h
	modified:   stub/src/mipsel.r3000-linux.elf-fold.S
	modified:   stub/src/powerpc-linux.elf-fold.S
	modified:   stub/src/powerpc64-linux.elf-fold.S
	modified:   stub/src/upxfd_android.c
	modified:   stub/src/upxfd_linux.c

src/stub/src/amd64-linux.elf-fold.S

@@ -58,6 +58,8 @@ __NR_close= 3
 __NR_mmap=      9
 __NR_mprotect= 10
 __NR_munmap=   11
+__NR_msync=    26  // 0x1a
+  MS_SYNC= 4
 __NR_brk=      12
 __NR_memfd_create= 0x13f  // 319
 __NR_ftruncate= 0x4d  // 77
@@ -270,6 +272,8 @@ Punmap: .globl Punmap  // page-align the lo end
         add %rax,%arg2
 munmap: .globl munmap
         movb $ __NR_munmap,%al; 5: jmp 5f
+msync: .globl msync
+        movb $__NR_msync,%al; 5: jmp 5f
 exit: .globl exit
         movb $ __NR_exit,%al; 5: jmp 5f
 brk: .globl brk

src/stub/src/amd64-linux.elf-main2.c

@@ -38,6 +38,7 @@
 // such as Ubuntu-20.04, Linux kernel 5.15.0-67, #74-20.04.1, 2023-02-22
 extern int upxfd_create(char const *tag, unsigned flags);
 #define MFD_EXEC 0x0010
+#define MS_SYNC 4
 
 extern void *memcpy(void *dst, void const *src, size_t n);
 // Pprotect is mprotect but uses page-aligned address (Linux requirement)
@@ -242,11 +243,12 @@ ERR_LAB
         xo->buf  += h.sz_unc;
         xo->size -= h.sz_unc;
     }
+    DPRINTF("unpackExtent done xo->buf=%%p\\n", xo->buf);
 }
 
 #if defined(__x86_64__)  //{
 static void *
-make_hatch_x86_64(
+make_hatch(
     ElfW(Phdr) const *const phdr,
     char *next_unc,
     unsigned const frag_mask
@@ -277,7 +279,7 @@ make_hatch_x86_64(
 }
 #elif defined(__powerpc64__)  //}{
 static void *
-make_hatch_ppc64(
+make_hatch(
     ElfW(Phdr) const *const phdr,
     char *next_unc,
     unsigned const frag_mask
@@ -314,7 +316,7 @@ make_hatch_ppc64(
 #define NBPI 4
 #define NINSTR 3
 static void *
-make_hatch_arm64(
+make_hatch(
     ElfW(Phdr) const *const phdr,
     char *next_unc,
     unsigned const frag_mask
@@ -606,13 +608,7 @@ do_xmap(
         }
 
         if (xi && phdr->p_flags & PF_X) {
-#if defined(__x86_64)  //{
+            void *const hatch = make_hatch(phdr, xo.buf, ~page_mask);
-            void *const hatch = make_hatch_x86_64(phdr, xo.buf, ~page_mask);
-#elif defined(__powerpc64__)  //}{
-            void *const hatch = make_hatch_ppc64(phdr, xo.buf, ~page_mask);
-#elif defined(__aarch64__)  //}{
-            void *const hatch = make_hatch_arm64(phdr, xo.buf, ~page_mask);
-#endif  //}
             if (0!=hatch) {
                 // Always update AT_NULL, especially for compressed PT_INTERP.
                 // Clearing lo bit of av is for i386 only; else is superfluous.
@@ -621,6 +617,7 @@ do_xmap(
 
             // SELinux: Map the contents of mfd as per *phdr.
             DPRINTF("hatch protect addr=%%p  mlen=%%p\\n", addr, mlen);
+            msync(addr, mlen, MS_SYNC); // be sure file gets de-compressed bytes
             munmap(addr, mlen);  // toss the VMA that has PROT_WRITE
             if (addr != mmap(addr, mlen, prot, MAP_FIXED|MAP_SHARED, mfd, 0)) {
                 err_exit(9);

src/stub/src/arm.v4a-linux.elf-fold.S

@@ -438,6 +438,7 @@ __NR_fsync=    118 + __NR_SYSCALL_BASE
 __NR_fdatasync=148 + __NR_SYSCALL_BASE
 __NR_uname=    122 + __NR_SYSCALL_BASE  // 0x7a
 __NR_mprotect= 125 + __NR_SYSCALL_BASE  // 0x7d
+__NR_msync=    144 + __NR_SYSCALL_BASE  // 0x90
 __NR_mmap2=    192 + __NR_SYSCALL_BASE  // 0xc0
 __NR_fstat=    197 + __NR_SYSCALL_BASE  // 0xc5
 __NR_memfd_create= 385 + __NR_SYSCALL_BASE  // 0x181

src/stub/src/arm64-linux.elf-entry.S

@@ -243,7 +243,7 @@ wmeth  .req w4
         mov arg6,#0  // beginning of file
         mov arg5w,mfd
         mov arg4w,#MAP_SHARED  // modes
-        mov arg3w,#PROT_WRITE|PROT_READ|PROT_EXEC  // FIXME: add PROT_WRITE for DEBUG only
+        mov arg3w,#PROT_READ|PROT_EXEC  // FIXME: add PROT_WRITE for DEBUG only
         ldr arg2w,[xFOLD,#sz_unc + LBINFO - LxFOLD]
         str arg2,[sp,#F_LENU]
         mov arg1,#0  // addr (kernel chooses)

src/stub/src/arm64-linux.elf-fold.S

@@ -123,6 +123,9 @@ mmap_privanon: .globl mmap_privanon
         orr w3,w3,w6  // flags |= MAP_{PRIVATE|ANON}  [QNX vs Linux]
     // FALL THROUGH to mmap
 
+msync: .globl msync
+        do_sys __NR_msync; ret
+
         .globl mmap
 mmap:
         ldr x8,PAGE_MASK
@@ -468,6 +471,7 @@ __NR_readlink = 0x4e + __NR_SYSCALL_BASE  // 78
 
 __NR_mmap     = 0xde + __NR_SYSCALL_BASE  // 222
 __NR_mprotect = 0xe2 + __NR_SYSCALL_BASE  // 226
+__NR_msync    = 0xe3 + __NR_SYSCALL_BASE  // 227
 __NR_munmap   = 0xd7 + __NR_SYSCALL_BASE  // 215
 __NR_memfd_create= 0x117 + __NR_SYSCALL_BASE  // 279
 __NR_ftruncate= 0x2e + __NR_SYSCALL_BASE  // 46

src/stub/src/i386-linux.elf-main2.c

@@ -37,10 +37,12 @@
 #define NO_WANT_CLOSE 1
 #define NO_WANT_EXIT 1
 #define NO_WANT_MPROTECT 1
+#define NO_WANT_MSYNC 1
 #define NO_WANT_WRITE 1
 #include "include/linux.h"
 
 #define MFD_EXEC 0x0010
+#define MS_SYNC 4
 #define nullptr 0
 
 extern void *memcpy(void *dst, void const *src, size_t n);
@@ -51,6 +53,7 @@ extern unsigned Pprotect(void *, size_t, unsigned);
 extern size_t Pwrite(unsigned, void const *, size_t);
 extern ssize_t write(int, void const *, size_t);
 extern int munmap(void *, size_t);
+extern int msync(void const *, size_t, unsigned);
 extern int close(int);
 extern void exit(int code) __attribute__ ((__noreturn__));
 #  define mmap_privanon(addr,len,prot,flgs) mmap((addr),(len),(prot), \
@@ -286,10 +289,11 @@ static char *
 make_hatch(
     ElfW(Phdr) const *const phdr,
     char *next_unc,
-    unsigned frag_mask
+    unsigned page_mask
 )
 {
     unsigned *hatch = 0;
+    unsigned frag_mask = ~page_mask;
     unsigned code[2] = {
         0x586180cd,  // int #0x80; popa; pop %eax
         0x90e0ff3e,  // notrack jmp *%eax; nop
@@ -324,10 +328,11 @@ static void *
 make_hatch(
     ElfW(Phdr) const *const phdr,
     char *next_unc,
-    unsigned frag_mask
+    unsigned page_mask
 )
 {
     unsigned const sys_munmap = get_sys_munmap();
+    unsigned frag_mask = ~page_mask;
     unsigned code[2] = {
         sys_munmap,  // syscall __NR_unmap
         0xe8bd8003,  // ldmia sp!,{r0,r1,pc}
@@ -361,8 +366,9 @@ static void *
 make_hatch(
     ElfW(Phdr) const *const phdr,
     char *next_unc,
-    unsigned frag_mask)
+    unsigned page_mask)
 {
+    unsigned frag_mask = ~page_mask;
     unsigned code[3];
     // avoid gcc initializing array by copying .rodata
     code[0] = 0x0000000c;  // syscall
@@ -401,8 +407,9 @@ static void *
 make_hatch(
     ElfW(Phdr) const *const phdr,
     char *next_unc,
-    unsigned frag_mask)
+    unsigned page_mask)
 {
+    unsigned frag_mask = ~page_mask;
     unsigned code[2];
     // avoid gcc initializing array by copying .rodata
     code[0] = 0x44000002;  // sc
@@ -702,7 +709,7 @@ do_xmap(
         }
 
         if (xi && phdr->p_flags & PF_X) {
-            char *hatch = make_hatch(phdr, xo.buf, ~page_mask);
+            char *hatch = make_hatch(phdr, xo.buf, page_mask);
             if (0!=hatch) {
                 // Always update AT_NULL, especially for compressed PT_INTERP.
                 // Clearing lo bit of av is for i386 only; else is superfluous.
@@ -711,6 +718,7 @@ do_xmap(
 
             // SELinux: Map the contents of mfd as per *phdr.
             DPRINTF("hatch protect addr=%%p  mlen=%%p\\n", addr, mlen);
+            msync(addr, mlen, MS_SYNC);  // be sure file gets de-compressed bytes
             munmap(addr, mlen);  // toss the VMA that has PROT_WRITE
             if (addr != mmap(addr, mlen, prot, MAP_FIXED|MAP_SHARED, mfd, 0)) {
                 err_exit(9);

src/stub/src/include/linux.h

@@ -186,6 +186,7 @@ struct timespec {
 #define __NR_uname              122
 #define __NR_adjtimex           124
 #define __NR_mprotect           125
+#define __NR_msync              144
 #define __NR_nanosleep          162
 #define __NR_memfd_create       356  /*0x164*/
 
@@ -363,6 +364,7 @@ static inline _syscall3(off_t,lseek,int,fd,off_t,offset,int,whence)
 static inline _syscall2(int,memfd_create,char const *,name,unsigned,flags);
 static inline _syscall2(int,mkdir,char const *,name,unsigned,mode);
 static inline _syscall3(int,mprotect,void *,addr,size_t,len,int,prot)
+static inline _syscall3(int,msync,void const *,addr,size_t,len,unsigned,flags)
 static inline _syscall2(int,munmap,void *,start,size_t,length)
 static inline _syscall2(int,nanosleep,const struct timespec *,rqtp,struct timespec *,rmtp)
 static inline _syscall3(int,open,const char *,file,int,flag,int,mode)
@@ -527,7 +529,25 @@ static int mprotect(void const *addr, size_t len, int prot)
     );
     return v0;
 }
-#endif  /* NO_WANT_MPROTECT */
+#endif  /*} NO_WANT_MPROTECT */
+
+#ifndef NO_WANT_MSYNC  /*{*/
+static int msync(void const *addr, size_t len, int prot)
+{
+#define __NR_msync (144+ 4000)
+    register void const *const a0 asm("a0") = addr;
+    register size_t      const a1 asm("a1") = len;
+    register int         const a2 asm("a2") = prot;
+    register size_t            v0 asm("v0") = __NR_msync;
+    __asm__ __volatile__(
+        "bal sysgo"
+    : "+r"(v0)
+    : "r"(a0), "r"(a1), "r"(a2)
+    : "a3", "ra"
+    );
+    return v0;
+}
+#endif  /*} NO_WANT_MSYNC */
 
 #ifndef NO_WANT_OPEN  /*{*/
 static ssize_t open(char const *path, int kind, int mode)
@@ -577,6 +597,7 @@ off_t lseek(int fd, off_t offset, int whence);
 int memfd_create(char const *, unsigned);
 int munmap(void *, size_t);
 int mprotect(void const *, size_t, int);
+int msync(void const *, size_t, unsigned);
 int open(char const *, int, int);
 int openat(int fd, char const *, unsigned, unsigned);
 ssize_t read(int, void *, size_t);

src/stub/src/mipsel.r3000-linux.elf-fold.S

@@ -69,6 +69,7 @@ __NR_memfd_create=354+__NR_Linux
 __NR_mmap=       90+ __NR_Linux
 __NR_mkdir=      39+ __NR_Linux
 __NR_mprotect = 125+ __NR_Linux
+__NR_msync =    144 + __NR_Linux
 __NR_munmap   =  91+ __NR_Linux
 __NR_oldstat=    18+ __NR_Linux
 __NR_open     =   5+ __NR_Linux
@@ -416,6 +417,8 @@ mkdir: .globl mkdir
         b sysgo; li v0,__NR_mkdir
 mprotect: .globl mprotect
         b sysgo; li v0,__NR_mprotect
+msync: .globl msync
+        b sysgo; li v0,__NR_msync
 munmap: .globl munmap
         b sysgo; li v0,__NR_munmap
 stat: .globl stat

src/stub/src/powerpc-linux.elf-fold.S

@@ -282,6 +282,7 @@ SYS_mmap=      90
 SYS_munmap=    91
 SYS_ftruncate= 93
 SYS_mprotect= 125
+SYS_msync=    144
 SYS_memfd_create= 360
 
 exit: .globl exit
@@ -304,6 +305,8 @@ open: .globl open
         li r0,SYS_open;     5: b 5f
 mprotect: .globl mprotect
         li 0,SYS_mprotect;  5: b 5f
+msync: .globl msync
+        li 0,SYS_msync;     5: b 5f
 munmap: .globl munmap
         li r0,SYS_munmap;   5: b sysgo
 mmap: .globl mmap

src/stub/src/powerpc64-linux.elf-fold.S

@@ -349,6 +349,7 @@ SYS_mmap=      90
 SYS_munmap=    91
 SYS_ftruncate= 93
 SYS_mprotect= 125
+SYS_msync=    144
 SYS_memfd_create= 360
 
 mmap: .globl mmap
@@ -365,6 +366,8 @@ munmap: .globl munmap
         li 0,SYS_munmap; 5: b 5f
 mprotect: .globl mprotect
         li 0,SYS_mprotect; 5: b 5f
+msync: .globl msync
+        li 0,SYS_msync; 5: b 5f
 memfd_create: .globl memfd_create
         li r0,SYS_memfd_create; 5: b 5f
 ftruncate: .globl ftruncate

src/stub/src/upxfd_android.c

@@ -131,6 +131,7 @@ struct stat { // __NR_stat = 106 + NR_SYSCALL_BASE
 // We want to supersede in *.elf-fold.S, not use include/linux.h
 #define NO_WANT_CLOSE 1
 #define NO_WANT_MPROTECT 1
+#define NO_WANT_MSYNC 1
 #define NO_WANT_OPEN 1
 #define NO_WANT_READ 1
 #include "include/linux.h"  // syscalls; i386 inlines via "int 0x80"

src/stub/src/upxfd_linux.c

@@ -83,6 +83,7 @@ extern void my_bkpt(void const *, ...);
 #define NO_WANT_READ 1
 #define NO_WANT_CLOSE 1
 #define NO_WANT_MPROTECT 1
+#define NO_WANT_MSYNC 1
 #endif  //}
 #include "include/linux.h"  // syscall decls; i386 inlines via "int 0x80"