Add unified memory aub tests

Related-To: NEO-2998

Change-Id: I1d047bc98de3cafe50ed7f54bfd3a6b695ec4dbd
Signed-off-by: Jobczyk, Lukasz <lukasz.jobczyk@intel.com>

core/memory_manager/graphics_allocation.h

@@ -218,6 +218,7 @@ class GraphicsAllocation : public IDNode<GraphicsAllocation> {
     StorageInfo storageInfo = {};
 
     static constexpr uint32_t defaultBank = 0b1u;
+    static constexpr uint32_t allBanks = 0xffffffff;
 
   protected:
     constexpr static uint32_t objectNotResident = std::numeric_limits<uint32_t>::max();

core/page_fault_manager/cpu_page_fault_manager.cpp

@@ -9,6 +9,8 @@
 
 #include "core/helpers/debug_helpers.h"
 #include "core/helpers/ptr_math.h"
+#include "core/memory_manager/graphics_allocation.h"
+#include "core/memory_manager/unified_memory_manager.h"
 
 #include <mutex>
 
@@ -37,6 +39,7 @@ void PageFaultManager::moveAllocationToGpuDomain(void *ptr) {
     if (alloc != memoryData.end()) {
         auto &pageFaultData = alloc->second;
         if (pageFaultData.isInGpuDomain == false) {
+            this->setAubWritable(false, ptr, pageFaultData.unifiedMemoryManager);
             this->transferToGpu(ptr, pageFaultData.cmdQ);
             this->protectCPUMemoryAccess(ptr, pageFaultData.size);
             pageFaultData.isInGpuDomain = true;
@@ -50,6 +53,7 @@ void PageFaultManager::moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocs
         auto allocPtr = alloc.first;
         auto &pageFaultData = alloc.second;
         if (pageFaultData.unifiedMemoryManager == unifiedMemoryManager && pageFaultData.isInGpuDomain == false) {
+            this->setAubWritable(false, allocPtr, pageFaultData.unifiedMemoryManager);
             this->transferToGpu(allocPtr, pageFaultData.cmdQ);
             this->protectCPUMemoryAccess(allocPtr, pageFaultData.size);
             pageFaultData.isInGpuDomain = true;
@@ -64,6 +68,7 @@ bool PageFaultManager::verifyPageFault(void *ptr) {
         auto &pageFaultData = alloc.second;
         if (ptr >= allocPtr && ptr < ptrOffset(allocPtr, pageFaultData.size)) {
             this->allowCPUMemoryAccess(allocPtr, pageFaultData.size);
+            this->setAubWritable(true, allocPtr, pageFaultData.unifiedMemoryManager);
             this->transferToCpu(allocPtr, pageFaultData.size, pageFaultData.cmdQ);
             pageFaultData.isInGpuDomain = false;
             return true;
@@ -71,4 +76,9 @@ bool PageFaultManager::verifyPageFault(void *ptr) {
     }
     return false;
 }
+
+void PageFaultManager::setAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
+    auto gpuAlloc = unifiedMemoryManager->getSVMAlloc(ptr)->gpuAllocation;
+    gpuAlloc->setAubWritable(writable, GraphicsAllocation::allBanks);
+}
 } // namespace NEO

core/page_fault_manager/cpu_page_fault_manager.h

@@ -41,6 +41,7 @@ class PageFaultManager : public NonCopyableOrMovableClass {
     MOCKABLE_VIRTUAL bool verifyPageFault(void *ptr);
     MOCKABLE_VIRTUAL void transferToCpu(void *ptr, size_t size, void *cmdQ);
     MOCKABLE_VIRTUAL void transferToGpu(void *ptr, void *cmdQ);
+    MOCKABLE_VIRTUAL void setAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager);
 
     std::unordered_map<void *, PageFaultData> memoryData;
     SpinLock mtx;

core/unit_tests/page_fault_manager/cpu_page_fault_manager_tests.cpp

@@ -5,7 +5,11 @@
  *
  */
 
+#include "core/memory_manager/graphics_allocation.h"
+#include "core/memory_manager/unified_memory_manager.h"
+#include "core/unified_memory/unified_memory.h"
 #include "core/unit_tests/page_fault_manager/cpu_page_fault_manager_tests_fixture.h"
+#include "unit_tests/mocks/mock_memory_manager.h"
 
 using namespace NEO;
 
@@ -26,6 +30,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenInsertingAllocsThenAllo
     EXPECT_EQ(pageFaultManager->protectMemoryCalled, 0);
     EXPECT_EQ(pageFaultManager->transferToCpuCalled, 1);
     EXPECT_EQ(pageFaultManager->transferToGpuCalled, 0);
+    EXPECT_TRUE(pageFaultManager->isAubWritable);
 
     pageFaultManager->insertAllocation(alloc2, 20, reinterpret_cast<SVMAllocsManager *>(unifiedMemoryManager), cmdQ);
     EXPECT_EQ(pageFaultManager->memoryData.size(), 2u);
@@ -41,6 +46,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenInsertingAllocsThenAllo
     EXPECT_EQ(pageFaultManager->protectMemoryCalled, 0);
     EXPECT_EQ(pageFaultManager->transferToCpuCalled, 2);
     EXPECT_EQ(pageFaultManager->transferToGpuCalled, 0);
+    EXPECT_TRUE(pageFaultManager->isAubWritable);
 
     pageFaultManager->removeAllocation(alloc1);
     EXPECT_EQ(pageFaultManager->memoryData.size(), 1u);
@@ -100,6 +106,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenMovingToGpuDomainAllocs
     EXPECT_EQ(pageFaultManager->protectedMemoryAccessAddress, alloc1);
     EXPECT_EQ(pageFaultManager->protectedSize, 10u);
     EXPECT_EQ(pageFaultManager->transferToGpuAddress, alloc1);
+    EXPECT_FALSE(pageFaultManager->isAubWritable);
 }
 
 TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenMoveToGpuDomainThenTransferToGpuIsCalled) {
@@ -121,6 +128,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenMoveToGpuDomainThenTrans
     EXPECT_EQ(pageFaultManager->protectedMemoryAccessAddress, alloc);
     EXPECT_EQ(pageFaultManager->protectedSize, 10u);
     EXPECT_EQ(pageFaultManager->transferToGpuAddress, alloc);
+    EXPECT_FALSE(pageFaultManager->isAubWritable);
 }
 
 TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocInGpuDomainWhenMovingToGpuDomainThenNothingIsCalled) {
@@ -139,6 +147,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocInGpuDomainWhenMovingToGpuDo
     EXPECT_EQ(pageFaultManager->protectMemoryCalled, 0);
     EXPECT_EQ(pageFaultManager->transferToCpuCalled, 1);
     EXPECT_EQ(pageFaultManager->transferToGpuCalled, 0);
+    EXPECT_TRUE(pageFaultManager->isAubWritable);
 }
 
 TEST_F(PageFaultManagerTest, whenMovingToGpuDomainUntrackedAllocThenNothingIsCalled) {
@@ -152,6 +161,7 @@ TEST_F(PageFaultManagerTest, whenMovingToGpuDomainUntrackedAllocThenNothingIsCal
     EXPECT_EQ(pageFaultManager->protectMemoryCalled, 0);
     EXPECT_EQ(pageFaultManager->transferToCpuCalled, 0);
     EXPECT_EQ(pageFaultManager->transferToGpuCalled, 0);
+    EXPECT_TRUE(pageFaultManager->isAubWritable);
 }
 
 TEST_F(PageFaultManagerTest, givenHandlerRegisteredAndUntrackedPageFaultAddressWhenVerifyingThenFalseIsReturned) {
@@ -187,4 +197,24 @@ TEST_F(PageFaultManagerTest, givenTrackedPageFaultAddressWhenVerifyingThenProper
     EXPECT_EQ(pageFaultManager->accessAllowedSize, 10u);
     EXPECT_EQ(pageFaultManager->transferToCpuAddress, alloc1);
     EXPECT_EQ(pageFaultManager->transferToCpuSize, 10u);
+    EXPECT_TRUE(pageFaultManager->isAubWritable);
+}
+
+TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenSetAubWritableIsCalledThenAllocIsAubWritable) {
+    MockExecutionEnvironment executionEnvironment;
+    if (!executionEnvironment.getHardwareInfo()->capabilityTable.ftrSvm) {
+        GTEST_SKIP();
+    }
+
+    void *cmdQ = reinterpret_cast<void *>(0xFFFF);
+    auto memoryManager = std::make_unique<MockMemoryManager>(executionEnvironment);
+    auto unifiedMemoryManager = std::make_unique<SVMAllocsManager>(memoryManager.get());
+    void *alloc1 = unifiedMemoryManager->createSharedUnifiedMemoryAllocation(0, 10, SVMAllocsManager::UnifiedMemoryProperties(InternalMemoryType::SHARED_UNIFIED_MEMORY), cmdQ);
+
+    pageFaultManager->baseAubWritable(false, alloc1, unifiedMemoryManager.get());
+
+    auto gpuAlloc = unifiedMemoryManager->getSVMAlloc(alloc1)->gpuAllocation;
+    EXPECT_FALSE(gpuAlloc->isAubWritable(GraphicsAllocation::allBanks));
+
+    unifiedMemoryManager->freeSVMAlloc(alloc1);
 }

core/unit_tests/page_fault_manager/mock_cpu_page_fault_manager.h

@@ -37,6 +37,12 @@ class MockPageFaultManager : public PageFaultManager {
         transferToGpuCalled++;
         transferToGpuAddress = ptr;
     }
+    void setAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager) override {
+        isAubWritable = writable;
+    }
+    void baseAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
+        PageFaultManager::setAubWritable(writable, ptr, unifiedMemoryManager);
+    }
     void baseCpuTransfer(void *ptr, size_t size, void *cmdQ) {
         PageFaultManager::transferToCpu(ptr, size, cmdQ);
     }
@@ -55,6 +61,7 @@ class MockPageFaultManager : public PageFaultManager {
     size_t transferToCpuSize = 0;
     size_t accessAllowedSize = 0;
     size_t protectedSize = 0;
+    bool isAubWritable = true;
 };
 
 template <class T>

unit_tests/aub_tests/CMakeLists.txt

@@ -47,6 +47,7 @@ endif()
 target_include_directories(igdrcl_aub_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
 
 target_sources(igdrcl_aub_tests PRIVATE
+  ${NEO_SOURCE_DIR}/core/unit_tests/page_fault_manager/default_asan_options.cpp
   ${NEO_SOURCE_DIR}/runtime/gmm_helper/resource_info.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test_mode.h
 )

unit_tests/aub_tests/fixtures/CMakeLists.txt

@@ -13,5 +13,6 @@ target_sources(igdrcl_aub_tests PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}/hello_world_fixture.h
   ${CMAKE_CURRENT_SOURCE_DIR}/run_kernel_fixture.h
   ${CMAKE_CURRENT_SOURCE_DIR}/simple_arg_fixture.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_fixture.h
 )
 add_subdirectories()

unit_tests/aub_tests/fixtures/unified_memory_fixture.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2019 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "core/memory_manager/memory_constants.h"
+#include "runtime/api/api.h"
+#include "unit_tests/aub_tests/fixtures/aub_fixture.h"
+
+namespace NEO {
+class UnifiedMemoryAubFixture : public AUBFixture {
+  public:
+    using AUBFixture::TearDown;
+
+    cl_int retVal = CL_SUCCESS;
+    const size_t dataSize = MemoryConstants::megaByte;
+    bool skipped = false;
+
+    void SetUp() override {
+        AUBFixture::SetUp(nullptr);
+        if (!platformImpl->peekExecutionEnvironment()->memoryManager->getPageFaultManager()) {
+            skipped = true;
+            GTEST_SKIP();
+        }
+    }
+
+    void *allocateUSM(InternalMemoryType type) {
+        void *ptr = nullptr;
+        if (!this->skipped) {
+            switch (type) {
+            case DEVICE_UNIFIED_MEMORY:
+                ptr = clDeviceMemAllocINTEL(this->context, this->device.get(), nullptr, dataSize, 0, &retVal);
+                break;
+            case HOST_UNIFIED_MEMORY:
+                ptr = clHostMemAllocINTEL(this->context, nullptr, dataSize, 0, &retVal);
+                break;
+            case SHARED_UNIFIED_MEMORY:
+                ptr = clSharedMemAllocINTEL(this->context, this->device.get(), nullptr, dataSize, 0, &retVal);
+                break;
+            default:
+                ptr = new char[dataSize];
+                break;
+            }
+            EXPECT_EQ(retVal, CL_SUCCESS);
+            EXPECT_NE(ptr, nullptr);
+        }
+        return ptr;
+    }
+
+    void freeUSM(void *ptr, InternalMemoryType type) {
+        if (!this->skipped) {
+            switch (type) {
+            case DEVICE_UNIFIED_MEMORY:
+            case HOST_UNIFIED_MEMORY:
+            case SHARED_UNIFIED_MEMORY:
+                retVal = clMemFreeINTEL(this->context, ptr);
+                break;
+            default:
+                delete[] static_cast<char *>(ptr);
+                break;
+            }
+            EXPECT_EQ(retVal, CL_SUCCESS);
+        }
+    }
+
+    void writeToUsmMemory(std::vector<char> data, void *ptr, InternalMemoryType type) {
+        if (!this->skipped) {
+            switch (type) {
+            case DEVICE_UNIFIED_MEMORY:
+                retVal = clEnqueueMemcpyINTEL(this->pCmdQ, true, ptr, data.data(), dataSize, 0, nullptr, nullptr);
+                break;
+            default:
+                std::copy(data.begin(), data.end(), static_cast<char *>(ptr));
+                break;
+            }
+            EXPECT_EQ(retVal, CL_SUCCESS);
+        }
+    }
+};
+} // namespace NEO

unit_tests/aub_tests/unified_memory/CMakeLists.txt

@@ -0,0 +1,11 @@
+#
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+target_sources(igdrcl_aub_tests PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+  ${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_aub_tests.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_copy_aub_tests.cpp
+)

unit_tests/aub_tests/unified_memory/unified_memory_aub_tests.cpp

@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2019 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "test.h"
+#include "unit_tests/aub_tests/fixtures/unified_memory_fixture.h"
+
+namespace NEO {
+
+class UnifiedMemoryAubTest : public UnifiedMemoryAubFixture,
+                             public ::testing::Test {
+  public:
+    using UnifiedMemoryAubFixture::TearDown;
+
+    std::vector<char> values;
+
+    void SetUp() override {
+        UnifiedMemoryAubFixture::SetUp();
+        values = std::vector<char>(dataSize, 11);
+    };
+};
+
+HWTEST_F(UnifiedMemoryAubTest, givenDeviceMemoryAllocWhenWriteIntoItThenValuesMatch) {
+    auto unifiedMemoryType = InternalMemoryType::DEVICE_UNIFIED_MEMORY;
+    auto unifiedMemoryPtr = allocateUSM(unifiedMemoryType);
+    writeToUsmMemory(values, unifiedMemoryPtr, unifiedMemoryType);
+
+    expectMemory<FamilyType>(unifiedMemoryPtr, values.data(), dataSize);
+
+    freeUSM(unifiedMemoryPtr, unifiedMemoryType);
+}
+
+HWTEST_F(UnifiedMemoryAubTest, givenSharedMemoryAllocWhenWriteIntoCPUPartThenValuesMatchAfterUsingAllocAsKernelParam) {
+    auto unifiedMemoryType = InternalMemoryType::SHARED_UNIFIED_MEMORY;
+    auto unifiedMemoryPtr = allocateUSM(unifiedMemoryType);
+
+    writeToUsmMemory(values, unifiedMemoryPtr, unifiedMemoryType);
+
+    expectNotEqualMemory<FamilyType>(unifiedMemoryPtr, values.data(), dataSize);
+
+    auto mockPtr = std::make_unique<char[]>(dataSize);
+    retVal = clEnqueueMemcpyINTEL(this->pCmdQ, true, mockPtr.get(), unifiedMemoryPtr, dataSize, 0, nullptr, nullptr);
+    EXPECT_EQ(retVal, CL_SUCCESS);
+
+    expectMemory<FamilyType>(unifiedMemoryPtr, values.data(), dataSize);
+
+    freeUSM(unifiedMemoryPtr, unifiedMemoryType);
+}
+
+HWTEST_F(UnifiedMemoryAubTest, givenSharedMemoryAllocWhenWriteIntoGPUPartThenValuesMatchAfterUsingAlloc) {
+    auto unifiedMemoryType = InternalMemoryType::SHARED_UNIFIED_MEMORY;
+
+    auto unifiedMemoryPtr = allocateUSM(unifiedMemoryType);
+    std::vector<char> input(dataSize, 11);
+
+    retVal = clEnqueueMemcpyINTEL(this->pCmdQ, true, unifiedMemoryPtr, input.data(), dataSize, 0, nullptr, nullptr);
+    EXPECT_EQ(retVal, CL_SUCCESS);
+
+    expectNotEqualMemory<FamilyType>(unifiedMemoryPtr, unifiedMemoryPtr, dataSize);
+    expectMemory<FamilyType>(unifiedMemoryPtr, input.data(), dataSize);
+
+    auto mockRead = reinterpret_cast<char *>(unifiedMemoryPtr)[0];
+    mockRead = 0;
+
+    expectMemory<FamilyType>(unifiedMemoryPtr, unifiedMemoryPtr, dataSize);
+
+    freeUSM(unifiedMemoryPtr, unifiedMemoryType);
+}
+} // namespace NEO

unit_tests/aub_tests/unified_memory/unified_memory_copy_aub_tests.cpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2019 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "test.h"
+#include "unit_tests/aub_tests/fixtures/unified_memory_fixture.h"
+
+namespace NEO {
+class UnifiedMemoryCopyAubTest : public UnifiedMemoryAubFixture,
+                                 public ::testing::TestWithParam<std::tuple<InternalMemoryType, InternalMemoryType>> {
+  public:
+    void *srcPtr, *dstPtr;
+    InternalMemoryType srcMemoryType, dstMemoryType;
+    std::vector<char> srcValues, dstValues;
+
+    void SetUp() override {
+        UnifiedMemoryAubFixture::SetUp();
+
+        srcMemoryType = std::get<0>(GetParam());
+        dstMemoryType = std::get<1>(GetParam());
+
+        srcPtr = this->allocateUSM(srcMemoryType);
+        dstPtr = this->allocateUSM(dstMemoryType);
+
+        srcValues = std::vector<char>(dataSize, 11);
+        dstValues = std::vector<char>(dataSize, 22);
+
+        this->writeToUsmMemory(srcValues, srcPtr, srcMemoryType);
+        this->writeToUsmMemory(dstValues, dstPtr, dstMemoryType);
+    }
+
+    void TearDown() override {
+        this->freeUSM(srcPtr, srcMemoryType);
+        this->freeUSM(dstPtr, dstMemoryType);
+        UnifiedMemoryAubFixture::TearDown();
+    }
+};
+
+HWTEST_P(UnifiedMemoryCopyAubTest, givenTwoUnifiedMemoryAllocsWhenCopyingOneToAnotherThenValuesMatch) {
+    clEnqueueMemcpyINTEL(this->pCmdQ, true, dstPtr, srcPtr, dataSize, 0, nullptr, nullptr);
+    expectMemory<FamilyType>(dstPtr, srcValues.data(), dataSize);
+}
+
+InternalMemoryType memoryTypes[] = {InternalMemoryType::HOST_UNIFIED_MEMORY,
+                                    InternalMemoryType::DEVICE_UNIFIED_MEMORY,
+                                    InternalMemoryType::SHARED_UNIFIED_MEMORY,
+                                    InternalMemoryType::NOT_SPECIFIED};
+
+INSTANTIATE_TEST_CASE_P(UnifiedMemoryCopyAubTest,
+                        UnifiedMemoryCopyAubTest,
+                        ::testing::Combine(::testing::ValuesIn(memoryTypes),
+                                           ::testing::ValuesIn(memoryTypes)));
+} // namespace NEO