coverage: Enable coverage reports

.coveragerc

@@ -1,50 +0,0 @@
-[run]
-branch = True
-concurrency =
-    multiprocessing
-data_file = .coverage/coverage
-parallel = True
-source =
-    meson.py
-    mesonbuild/
-    mesonconf.py
-    mesonintrospect.py
-    mesonrewriter.py
-    mesontest.py
-    run_cross_test.py
-    run_project_tests.py
-    run_tests.py
-    run_unittests.py
-
-# Aliases to /root/ are needed because Travis runs in docker at /root/.
-[paths]
-meson =
-    meson.py
-    /root/meson.py
-mesonbuild =
-    mesonbuild/
-    /root/mesonbuild/
-mesonconf =
-    mesonconf.py
-    /root/mesonconf.py
-mesonintrospect =
-    mesonintrospect.py
-    /root/mesonintrospect.py
-mesonrewriter =
-    mesonrewriter.py
-    /root/mesonrewriter.py
-mesontest =
-    mesontest.py
-    /root/mesontest.py
-run_cross_test =
-    run_cross_test.py
-    /root/run_cross_test.py
-run_project_tests =
-    run_project_tests.py
-    /root/run_project_tests.py
-run_tests =
-    run_tests.py
-    /root/run_tests.py
-run_unittests =
-    run_unittests.py
-    /root/run_unittests.py

.github/workflows/cygwin.yml

@@ -68,13 +68,13 @@ jobs:
       - name: Run pip
         run: |
           export PATH=/usr/bin:/usr/local/bin:$(cygpath ${SYSTEMROOT})/system32
-          python3 -m pip --disable-pip-version-check install gcovr jsonschema pefile pytest pytest-xdist
+          python3 -m pip --disable-pip-version-check install gcovr jsonschema pefile pytest pytest-xdist coverage codecov
         shell: C:\cygwin\bin\bash.exe --noprofile --norc -o igncr -eo pipefail '{0}'
 
       - name: Run tests
         run: |
           export PATH=/usr/bin:/usr/local/bin:$(cygpath ${SYSTEMROOT})/system32
-          python3 run_tests.py --backend=ninja
+          python3 ./tools/run_with_cov.py run_tests.py --backend=ninja
         env:
           # Cygwin's static boost installation is broken (some static library
           # variants such as boost_thread are not present)
@@ -87,3 +87,6 @@ jobs:
           path: meson-test-run.*
         # test log should be saved on failure
         if: ${{ !cancelled() }}
+
+      - name: Upload coverage report
+        run: ./ci/upload_cov.sh "${{ matrix.NAME }}"

.github/workflows/macos.yml

@@ -24,7 +24,7 @@ jobs:
         python-version: '3.x'
     - run: |
         python -m pip install --upgrade pip
-        python -m pip install pytest pytest-xdist jsonschema
+        python -m pip install pytest pytest-xdist jsonschema coverage codecov
     - run: brew install pkg-config ninja llvm qt@5
     - env:
         CPPFLAGS: "-I/usr/local/include"
@@ -35,7 +35,10 @@ jobs:
         export SDKROOT="$(xcodebuild -version -sdk macosx Path)"
         export PATH="$HOME/tools:/usr/local/opt/qt@5/bin:$PATH:$(brew --prefix llvm)/bin"
         export PKG_CONFIG_PATH="/usr/local/opt/qt@5/lib/pkgconfig:$PKG_CONFIG_PATH"
-        ./run_unittests.py
+        ./tools/run_with_cov.py ./run_unittests.py
+    - name: Upload coverage report
+      run: ./ci/upload_cov.sh "appleclang [unit tests]"
+
 
   project-tests-appleclang:
     runs-on: macos-latest
@@ -50,7 +53,7 @@ jobs:
     - run: |
         python3 -m pip install --upgrade setuptools
         python3 -m pip install --upgrade pip
-        python3 -m pip install cython
+        python3 -m pip install cython coverage codecov
     - env:
         CPPFLAGS: "-I/usr/local/include"
         LDFLAGS: "-L/usr/local/lib"
@@ -61,7 +64,9 @@ jobs:
         export SDKROOT="$(xcodebuild -version -sdk macosx Path)"
         export PATH="$HOME/tools:/usr/local/opt/qt@5/bin:$PATH:$(brew --prefix llvm)/bin"
         export PKG_CONFIG_PATH="/usr/local/opt/qt@5/lib/pkgconfig:$PKG_CONFIG_PATH"
-        ./run_project_tests.py --backend=ninja
+        ./tools/run_with_cov.py ./run_project_tests.py --backend=ninja
+    - name: Upload coverage report
+      run: ./ci/upload_cov.sh "appleclang [project tests; unity=${{ matrix.unity }}]"
 
   Qt4macos:
     runs-on: macos-latest

.github/workflows/msys2.yml

@@ -71,7 +71,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python3 -m pip --disable-pip-version-check install gcovr jsonschema pefile pytest pytest-xdist
+          python3 -m pip --disable-pip-version-check install gcovr jsonschema pefile pytest pytest-xdist coverage codecov
 
       - name: Run Tests
         run: |
@@ -89,9 +89,12 @@ jobs:
             pacman --noconfirm --needed -S mingw-w64-${{ matrix.MSYS2_ARCH }}-${{ matrix.MSYS2_CURSES }}
           fi
 
-          MSYSTEM= python3 run_tests.py --backend=ninja
+          MSYSTEM= python3 ./tools/run_with_cov.py run_tests.py --backend=ninja
 
       - uses: actions/upload-artifact@v2
         with:
           name: ${{ matrix.NAME }}
           path: meson-test-run.*
+
+      - name: Upload coverage report
+        run: ./ci/upload_cov.sh "${{ matrix.NAME }}"

.github/workflows/nonative.yml

@@ -26,6 +26,9 @@ jobs:
     - run: |
         apt-get -y purge clang gcc gdc
         apt-get -y autoremove
+        python3 -m pip install coverage codecov
     - uses: actions/checkout@v2
     - name: Run tests
-      run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./run_tests.py $CI_ARGS --cross ubuntu-armhf.json --cross-only'
+      run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./tools/run_with_cov.py ./run_tests.py $CI_ARGS --cross ubuntu-armhf.json --cross-only'
+    - name: Upload coverage report
+      run: ./ci/upload_cov.sh "Ubuntu nonnative"

.github/workflows/os_comp.yml

@@ -40,7 +40,9 @@ jobs:
       # All environment variables are stored inside the docker image in /ci/env_vars.sh
       # They are defined in the `env` section in each image.json. CI_ARGS should be set
       # via the `args` array ub the image.json
-      run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./run_tests.py $CI_ARGS'
+      run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./tools/run_with_cov.py ./run_tests.py $CI_ARGS'
+    - name: Upload coverage report
+      run: ./ci/upload_cov.sh "OS Comp [${{ matrix.cfg.name }}]"
 
   ubuntu-rolling:
     name: 'Ubuntu Rolling'
@@ -106,4 +108,7 @@ jobs:
           update-alternatives --set i686-w64-mingw32-gcc   /usr/bin/i686-w64-mingw32-gcc-posix
           update-alternatives --set i686-w64-mingw32-g++   /usr/bin/i686-w64-mingw32-g++-posix
 
-          ./run_tests.py $RUN_TESTS_ARGS -- $MESON_ARGS
+          ./tools/run_with_cov.py ./run_tests.py $RUN_TESTS_ARGS -- $MESON_ARGS
+
+      - name: Upload coverage report
+        run: ./ci/upload_cov.sh "Ubuntu [${{ matrix.cfg.CC }} ${{ matrix.cfg.RUN_TESTS_ARGS }} ${{ matrix.cfg.MESON_ARGS }}]"

.github/workflows/unusedargs_missingreturn.yml

@@ -45,7 +45,10 @@ jobs:
       run: |
         sudo apt update -yq
         sudo apt install -yq --no-install-recommends g++ gfortran ninja-build gobjc gobjc++
-    - run: python run_project_tests.py --only cmake common fortran platform-linux "objective c" "objective c++"
+        python -m pip install coverage codecov
+    - run: ./tools/run_with_cov.py run_project_tests.py --only cmake common fortran platform-linux "objective c" "objective c++"
+    - name: Upload coverage report
+      run: ./ci/upload_cov.sh "UnusedMissingReturn"
 
   windows:
     runs-on: windows-latest
@@ -55,10 +58,13 @@ jobs:
       with:
         python-version: '3.x'
 
-    - run: pip install ninja pefile
+    - run: pip install ninja pefile coverage codecov
 
-    - run: python run_project_tests.py --only platform-windows
+    - run: python ./tools/run_with_cov.py run_project_tests.py --only platform-windows
       env:
         CC: gcc
         CXX: g++
         FC: gfortran
+
+    - name: Upload coverage report
+      run: ./ci/upload_cov.sh "UnusedMissingReturn Windows"

.gitignore

@@ -8,7 +8,8 @@
 /.vscode
 
 __pycache__
-.coverage
+/.coverage/
+/.coveragerc
 /install dir
 /work area
 

azure-pipelines.yml

@@ -9,6 +9,7 @@ trigger:
 
 variables:
   CI: 1
+  SOURCE_VERSION: $(Build.SourceVersion)
 
 jobs:
 

ci/run.ps1

@@ -68,11 +68,26 @@ python --version
 
 # Needed for running unit tests in parallel.
 echo ""
-python -m pip --disable-pip-version-check install --upgrade pefile pytest-xdist jsonschema
+python -m pip --disable-pip-version-check install --upgrade pefile pytest-xdist jsonschema coverage
 
 echo ""
 echo "=== Start running tests ==="
 # Starting from VS2019 Powershell(?) will fail the test run
 # if it prints anything to stderr. Python's test runner
 # does that by default so we need to forward it.
-cmd /c "python 2>&1 run_tests.py --backend $env:backend $env:extraargs"
+cmd /c "python 2>&1 ./tools/run_with_cov.py  run_tests.py --backend $env:backend $env:extraargs"
+
+echo ""
+echo ""
+echo "=== Gathering coverage report ==="
+echo ""
+
+python3 -m coverage combine
+python3 -m coverage xml
+python3 -m coverage report
+
+# Currently codecov.py does not handle Azure, use this fork of a fork to get it
+# working without requireing a token
+git clone https://github.com/mensinda/codecov-python
+python3 -m pip install --ignore-installed ./codecov-python
+python3 -m codecov -f .coverage/coverage.xml -n "VS$env:compiler $env:arch $env:backend" -c $env:SOURCE_VERSION

ci/upload_cov.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "Combining coverage reports..."
+coverage combine
+
+echo "Generating XML report..."
+coverage xml
+
+echo "Printing report"
+coverage report
+
+echo "Uploading to codecov..."
+codecov -f .coverage/coverage.xml -n "$1"

ci/usercustomize.py

@@ -0,0 +1,19 @@
+# Copyright 2021 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used by coverage (see tools/run_with_cov.py) to enable coverage
+# reports in python subprocesses
+
+import coverage
+coverage.process_startup()

data/.coveragerc.in

@@ -0,0 +1,25 @@
+[run]
+branch      = True
+parallel    = True
+concurrency = multiprocessing
+data_file   = @ROOT@/.coverage/coverage
+source      = @ROOT@/mesonbuild/
+
+[report]
+exclude_lines =
+  if T.TYPE_CHECKING:
+
+[paths]
+mesonbuild =
+    mesonbuild/
+    __w/meson/meson/mesonbuild/
+    @ROOT@/mesonbuild/
+
+[html]
+directory = @ROOT@/.coverage/html
+
+[xml]
+output    = @ROOT@/.coverage/coverage.xml
+
+[json]
+output    = @ROOT@/.coverage/coverage.json

run_tests.py

@@ -20,7 +20,6 @@ import sys
 import time
 import shutil
 import subprocess
-import tempfile
 import platform
 import argparse
 import traceback
@@ -311,7 +310,6 @@ def print_system_info():
 def main():
     print_system_info()
     parser = argparse.ArgumentParser()
-    parser.add_argument('--cov', action='store_true')
     parser.add_argument('--backend', default=None, dest='backend',
                         choices=backendlist)
     parser.add_argument('--cross', default=[], dest='cross', action='append')
@@ -319,13 +317,6 @@ def main():
     parser.add_argument('--failfast', action='store_true')
     parser.add_argument('--no-unittests', action='store_true', default=False)
     (options, _) = parser.parse_known_args()
-    # Enable coverage early...
-    enable_coverage = options.cov
-    if enable_coverage:
-        os.makedirs('.coverage', exist_ok=True)
-        sys.argv.remove('--cov')
-        import coverage
-        coverage.process_startup()
     returncode = 0
     backend, _ = guess_backend(options.backend, shutil.which('msbuild'))
     no_unittests = options.no_unittests
@@ -349,52 +340,41 @@ def main():
     # Run tests
     # Can't pass arguments to unit tests, so set the backend to use in the environment
     env = os.environ.copy()
-    with tempfile.TemporaryDirectory() as temp_dir:
-        # Enable coverage on all subsequent processes.
-        if enable_coverage:
-            Path(temp_dir, 'usercustomize.py').open('w').write(
-                'import coverage\n'
-                'coverage.process_startup()\n')
-            env['COVERAGE_PROCESS_START'] = '.coveragerc'
-            if 'PYTHONPATH' in env:
-                env['PYTHONPATH'] = os.pathsep.join([temp_dir, env.get('PYTHONPATH')])
-            else:
-                env['PYTHONPATH'] = temp_dir
-        if not options.cross:
-            cmd = mesonlib.python_command + ['run_meson_command_tests.py', '-v']
+    if not options.cross:
+        cmd = mesonlib.python_command + ['run_meson_command_tests.py', '-v']
+        if options.failfast:
+            cmd += ['--failfast']
+        returncode += subprocess.call(cmd, env=env)
+        if options.failfast and returncode != 0:
+            return returncode
+        if no_unittests:
+            print('Skipping all unit tests.')
+            print(flush=True)
+            returncode = 0
+        else:
+            print(mlog.bold('Running unittests.'))
+            print(flush=True)
+            cmd = mesonlib.python_command + ['run_unittests.py', '--backend=' + backend.name, '-v']
             if options.failfast:
                 cmd += ['--failfast']
             returncode += subprocess.call(cmd, env=env)
             if options.failfast and returncode != 0:
                 return returncode
-            if no_unittests:
-                print('Skipping all unit tests.')
-                print(flush=True)
-                returncode = 0
-            else:
-                print(mlog.bold('Running unittests.'))
-                print(flush=True)
-                cmd = mesonlib.python_command + ['run_unittests.py', '--backend=' + backend.name, '-v']
-                if options.failfast:
-                    cmd += ['--failfast']
-                returncode += subprocess.call(cmd, env=env)
-                if options.failfast and returncode != 0:
-                    return returncode
-            cmd = mesonlib.python_command + ['run_project_tests.py'] + sys.argv[1:]
+        cmd = mesonlib.python_command + ['run_project_tests.py'] + sys.argv[1:]
+        returncode += subprocess.call(cmd, env=env)
+    else:
+        cross_test_args = mesonlib.python_command + ['run_cross_test.py']
+        for cf in options.cross:
+            print(mlog.bold(f'Running {cf} cross tests.'))
+            print(flush=True)
+            cmd = cross_test_args + ['cross/' + cf]
+            if options.failfast:
+                cmd += ['--failfast']
+            if options.cross_only:
+                cmd += ['--cross-only']
             returncode += subprocess.call(cmd, env=env)
-        else:
-            cross_test_args = mesonlib.python_command + ['run_cross_test.py']
-            for cf in options.cross:
-                print(mlog.bold(f'Running {cf} cross tests.'))
-                print(flush=True)
-                cmd = cross_test_args + ['cross/' + cf]
-                if options.failfast:
-                    cmd += ['--failfast']
-                if options.cross_only:
-                    cmd += ['--cross-only']
-                returncode += subprocess.call(cmd, env=env)
-                if options.failfast and returncode != 0:
-                    return returncode
+            if options.failfast and returncode != 0:
+                return returncode
     return returncode
 
 if __name__ == '__main__':

run_unittests.py

@@ -10302,6 +10302,13 @@ def main():
             pytest_args += ['--color=yes']
         pytest_args += ['./run_unittests.py']
         pytest_args += convert_args(sys.argv[1:])
+        # Always disable pytest-cov because we use a custom setup
+        try:
+            import pytest_cov # noqa: F401
+            print('Disabling pytest-cov')
+            pytest_args += ['-p' 'no:cov']
+        except ImportError:
+            pass
         return subprocess.run(python_command + ['-m', 'pytest'] + pytest_args).returncode
     except ImportError:
         print('pytest-xdist not found, using unittest instead')

tools/run_with_cov.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# Copyright 2021 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import coverage
+import os
+import sys
+from pathlib import Path
+
+root_path = Path(__file__).parent.parent.absolute()
+
+# Python magic so we can import mesonlib
+sys.path.append(root_path.as_posix())
+from mesonbuild import mesonlib
+
+def generate_coveragerc() -> Path:
+    i_file = (root_path / 'data' / '.coveragerc.in')
+    o_file = (root_path / '.coveragerc')
+    raw = i_file.read_text()
+    raw = raw.replace('@ROOT@', root_path.as_posix())
+    o_file.write_text(raw)
+    return o_file
+
+def main() -> int:
+    # Remove old run data
+    out_dir = root_path / '.coverage'
+    mesonlib.windows_proof_rmtree(out_dir.as_posix())
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    # Setup coverage
+    python_path = (root_path / 'ci').as_posix()
+    os.environ['PYTHONPATH'] = os.pathsep.join([python_path, os.environ.get('PYTHONPATH', '')])
+    os.environ['COVERAGE_PROCESS_START'] = generate_coveragerc().as_posix()
+    coverage.process_startup()
+
+    # Run the actual command
+    cmd = mesonlib.python_command + sys.argv[1:]
+    return subprocess.run(cmd, env=os.environ.copy()).returncode
+
+if __name__ == '__main__':
+    raise SystemExit(main())