diff --git a/opencl/test/unit_test/mocks/CMakeLists.txt b/opencl/test/unit_test/mocks/CMakeLists.txt index 0eb315529f..fd74d938ca 100644 --- a/opencl/test/unit_test/mocks/CMakeLists.txt +++ b/opencl/test/unit_test/mocks/CMakeLists.txt @@ -108,6 +108,8 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/mock_wddm.h ${CMAKE_CURRENT_SOURCE_DIR}/mock_wddm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/mock_wddm_residency_allocations_container.h + ${CMAKE_CURRENT_SOURCE_DIR}/mock_wddm_residency_logger.h + ${CMAKE_CURRENT_SOURCE_DIR}/mock_wddm_residency_logger_functions.h ${CMAKE_CURRENT_SOURCE_DIR}/wddm_mock_helpers.h ${IGDRCL_SRC_tests_mock_wddm} ) diff --git a/opencl/test/unit_test/mocks/mock_wddm.cpp b/opencl/test/unit_test/mocks/mock_wddm.cpp index 2c351da9b7..89093597aa 100644 --- a/opencl/test/unit_test/mocks/mock_wddm.cpp +++ b/opencl/test/unit_test/mocks/mock_wddm.cpp @@ -14,6 +14,7 @@ #include "opencl/test/unit_test/mock_gdi/mock_gdi.h" #include "opencl/test/unit_test/mocks/mock_wddm_residency_allocations_container.h" +#include "opencl/test/unit_test/mocks/mock_wddm_residency_logger.h" #include "gtest/gtest.h" @@ -27,14 +28,14 @@ WddmMock::~WddmMock() { EXPECT_EQ(0, reservedAddresses.size()); } -bool WddmMock::makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) { +bool WddmMock::makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t totalSize) { makeResidentResult.called++; makeResidentResult.handleCount = count; for (auto i = 0u; i < count; i++) { makeResidentResult.handlePack.push_back(handles[i]); } if (callBaseMakeResident) { - return makeResidentResult.success = Wddm::makeResident(handles, count, cantTrimFurther, numberOfBytesToTrim); + return makeResidentResult.success = Wddm::makeResident(handles, count, cantTrimFurther, numberOfBytesToTrim, totalSize); } else { makeResidentResult.success = makeResidentStatus; return makeResidentStatus; @@ -171,9 +172,9 @@ bool WddmMock::waitOnGPU(D3DKMT_HANDLE context) { return waitOnGPUResult.success = Wddm::waitOnGPU(context); } -void *WddmMock::lockResource(const D3DKMT_HANDLE &handle, bool applyMakeResidentPriorToLock) { +void *WddmMock::lockResource(const D3DKMT_HANDLE &handle, bool applyMakeResidentPriorToLock, size_t size) { lockResult.called++; - auto ptr = Wddm::lockResource(handle, applyMakeResidentPriorToLock); + auto ptr = Wddm::lockResource(handle, applyMakeResidentPriorToLock, size); lockResult.success = ptr != nullptr; lockResult.uint64ParamPassed = applyMakeResidentPriorToLock; return ptr; @@ -278,6 +279,16 @@ void WddmMock::waitOnPagingFenceFromCpu() { Wddm::waitOnPagingFenceFromCpu(); } +void WddmMock::createPagingFenceLogger() { + if (callBaseCreatePagingLogger) { + Wddm::createPagingFenceLogger(); + } else { + if (DebugManager.flags.WddmResidencyLogger.get()) { + residencyLogger = std::make_unique(device, pagingFenceAddress); + } + } +} + void *GmockWddm::virtualAllocWrapper(void *inPtr, size_t size, uint32_t flags, uint32_t type) { void *tmp = reinterpret_cast(virtualAllocAddress); size += MemoryConstants::pageSize; diff --git a/opencl/test/unit_test/mocks/mock_wddm.h b/opencl/test/unit_test/mocks/mock_wddm.h index 385e85d8c7..1e92b20291 100644 --- a/opencl/test/unit_test/mocks/mock_wddm.h +++ b/opencl/test/unit_test/mocks/mock_wddm.h @@ -28,6 +28,7 @@ constexpr auto virtualAllocAddress = is64bit ? 0x7FFFF0000000 : 0xFF000000; class WddmMock : public Wddm { public: using Wddm::adapterBDF; + using Wddm::createPagingFenceLogger; using Wddm::currentPagingFenceValue; using Wddm::dedicatedVideoMemory; using Wddm::device; @@ -39,13 +40,14 @@ class WddmMock : public Wddm { using Wddm::minAddress; using Wddm::pagingFenceAddress; using Wddm::pagingQueue; + using Wddm::residencyLogger; using Wddm::temporaryResources; using Wddm::wddmInterface; WddmMock(RootDeviceEnvironment &rootDeviceEnvironment); ~WddmMock(); - bool makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) override; + bool makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t totalSize) override; bool evict(const D3DKMT_HANDLE *handles, uint32_t num, uint64_t &sizeToTrim) override; bool mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_VIRTUAL_ADDRESS preferredAddress, D3DGPU_VIRTUAL_ADDRESS &gpuPtr) override; bool mapGpuVirtualAddress(WddmAllocation *allocation); @@ -64,7 +66,7 @@ class WddmMock : public Wddm { bool queryAdapterInfo() override; bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) override; bool waitOnGPU(D3DKMT_HANDLE context) override; - void *lockResource(const D3DKMT_HANDLE &handle, bool applyMakeResidentPriorToLock) override; + void *lockResource(const D3DKMT_HANDLE &handle, bool applyMakeResidentPriorToLock, size_t size) override; void unlockResource(const D3DKMT_HANDLE &handle) override; void kmDafLock(D3DKMT_HANDLE handle) override; bool isKmDafEnabled() const override; @@ -82,6 +84,7 @@ class WddmMock : public Wddm { PLATFORM *getGfxPlatform() { return gfxPlatform.get(); } uint64_t *getPagingFenceAddress() override; void waitOnPagingFenceFromCpu() override; + void createPagingFenceLogger() override; bool configureDeviceAddressSpace() { configureDeviceAddressSpaceResult.called++; @@ -130,6 +133,7 @@ class WddmMock : public Wddm { uint64_t mockPagingFence = 0u; bool makeResidentStatus = true; bool callBaseMakeResident = true; + bool callBaseCreatePagingLogger = true; }; struct GmockWddm : WddmMock { @@ -141,7 +145,7 @@ struct GmockWddm : WddmMock { void *virtualAllocWrapper(void *inPtr, size_t size, uint32_t flags, uint32_t type); uintptr_t virtualAllocAddress; - MOCK_METHOD4(makeResident, bool(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim)); + MOCK_METHOD5(makeResident, bool(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t totalSize)); MOCK_METHOD3(evict, bool(const D3DKMT_HANDLE *handles, uint32_t num, uint64_t &sizeToTrim)); MOCK_METHOD1(createAllocationsAndMapGpuVa, NTSTATUS(OsHandleStorage &osHandles)); diff --git a/opencl/test/unit_test/mocks/mock_wddm_residency_allocations_container.h b/opencl/test/unit_test/mocks/mock_wddm_residency_allocations_container.h index e854a1b5f0..67a8a66de4 100644 --- a/opencl/test/unit_test/mocks/mock_wddm_residency_allocations_container.h +++ b/opencl/test/unit_test/mocks/mock_wddm_residency_allocations_container.h @@ -22,9 +22,9 @@ class MockWddmResidentAllocationsContainer : public WddmResidentAllocationsConta MockWddmResidentAllocationsContainer(Wddm *wddm) : WddmResidentAllocationsContainer(wddm) {} virtual ~MockWddmResidentAllocationsContainer() = default; - MemoryOperationsStatus makeResidentResource(const D3DKMT_HANDLE &handle) override { + MemoryOperationsStatus makeResidentResource(const D3DKMT_HANDLE &handle, size_t size) override { makeResidentResult.called++; - makeResidentResult.operationSuccess = WddmResidentAllocationsContainer::makeResidentResource(handle); + makeResidentResult.operationSuccess = WddmResidentAllocationsContainer::makeResidentResource(handle, size); return makeResidentResult.operationSuccess; } diff --git a/opencl/test/unit_test/mocks/mock_wddm_residency_logger.h b/opencl/test/unit_test/mocks/mock_wddm_residency_logger.h new file mode 100644 index 0000000000..44f093748b --- /dev/null +++ b/opencl/test/unit_test/mocks/mock_wddm_residency_logger.h @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include "shared/source/os_interface/windows/wddm/wddm_residency_logger.h" + +namespace NEO { +struct MockWddmResidencyLogger : public WddmResidencyLogger { + using WddmResidencyLogger::endTime; + using WddmResidencyLogger::enterWait; + using WddmResidencyLogger::makeResidentCall; + using WddmResidencyLogger::pagingLog; + using WddmResidencyLogger::pendingMakeResident; + using WddmResidencyLogger::pendingTime; + using WddmResidencyLogger::waitStartTime; + + MockWddmResidencyLogger(D3DKMT_HANDLE device, VOID *fenceValueCpuVirtualAddress) + : WddmResidencyLogger(device, fenceValueCpuVirtualAddress) { + } +}; +} // namespace NEO diff --git a/opencl/test/unit_test/mocks/mock_wddm_residency_logger_functions.h b/opencl/test/unit_test/mocks/mock_wddm_residency_logger_functions.h new file mode 100644 index 0000000000..235d5b54bb --- /dev/null +++ b/opencl/test/unit_test/mocks/mock_wddm_residency_logger_functions.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include "shared/source/os_interface/windows/wddm/wddm_residency_logger_defs.h" + +namespace NEO { +namespace ResLog { +extern uint32_t mockFopenCalled; +extern uint32_t mockVfptrinfCalled; +extern uint32_t mockFcloseCalled; + +FILE *mockFopen(const char *filename, const char *mode) { + mockFopenCalled++; + return reinterpret_cast(0x40); +} + +int mockVfptrinf(FILE *stream, const char *format, va_list arg) { + mockVfptrinfCalled++; + return 0x10; +} + +int mockFclose(FILE *stream) { + mockFcloseCalled++; + return 0; +} +} // namespace ResLog +} // namespace NEO diff --git a/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp index 59757ab717..2652a03b9b 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp @@ -26,6 +26,7 @@ #include "opencl/test/unit_test/mocks/mock_gfx_partition.h" #include "opencl/test/unit_test/mocks/mock_gmm_resource_info.h" #include "opencl/test/unit_test/mocks/mock_memory_manager.h" +#include "opencl/test/unit_test/mocks/mock_wddm_residency_logger.h" #include "opencl/test/unit_test/os_interface/windows/mock_wddm_allocation.h" #include "opencl/test/unit_test/os_interface/windows/ult_dxgi_factory.h" #include "opencl/test/unit_test/os_interface/windows/wddm_fixture.h" @@ -459,7 +460,7 @@ TEST_F(Wddm20Tests, makeResidentNonResident) { EXPECT_TRUE(error); EXPECT_TRUE(allocation.getGpuAddress() != 0); - error = wddm->makeResident(allocation.getHandles().data(), allocation.getNumHandles(), false, nullptr); + error = wddm->makeResident(allocation.getHandles().data(), allocation.getNumHandles(), false, nullptr, allocation.getAlignedSize()); EXPECT_TRUE(error); uint64_t sizeToTrim; @@ -666,7 +667,7 @@ TEST_F(Wddm20Tests, makeResidentMultipleHandles) { gdi->getMakeResidentArg().NumAllocations = 0; gdi->getMakeResidentArg().AllocationList = nullptr; - bool error = wddm->makeResident(handles, 2, false, nullptr); + bool error = wddm->makeResident(handles, 2, false, nullptr, 0x1000); EXPECT_TRUE(error); EXPECT_EQ(2u, gdi->getMakeResidentArg().NumAllocations); @@ -681,7 +682,7 @@ TEST_F(Wddm20Tests, makeResidentMultipleHandlesWithReturnBytesToTrim) { gdi->getMakeResidentArg().NumBytesToTrim = 30; uint64_t bytesToTrim = 0; - bool success = wddm->makeResident(handles, 2, false, &bytesToTrim); + bool success = wddm->makeResident(handles, 2, false, &bytesToTrim, 0x1000); EXPECT_TRUE(success); EXPECT_EQ(gdi->getMakeResidentArg().NumBytesToTrim, bytesToTrim); @@ -891,29 +892,29 @@ using WddmLockWithMakeResidentTests = Wddm20Tests; TEST_F(WddmLockWithMakeResidentTests, givenAllocationThatDoesntNeedMakeResidentBeforeLockWhenLockThenDontStoreItOrCallMakeResident) { EXPECT_TRUE(mockTemporaryResources->resourceHandles.empty()); EXPECT_EQ(0u, wddm->makeResidentResult.called); - wddm->lockResource(ALLOCATION_HANDLE, false); + wddm->lockResource(ALLOCATION_HANDLE, false, 0x1000); EXPECT_TRUE(mockTemporaryResources->resourceHandles.empty()); EXPECT_EQ(0u, wddm->makeResidentResult.called); wddm->unlockResource(ALLOCATION_HANDLE); } TEST_F(WddmLockWithMakeResidentTests, givenAllocationThatNeedsMakeResidentBeforeLockWhenLockThenCallBlockingMakeResident) { - wddm->lockResource(ALLOCATION_HANDLE, true); + wddm->lockResource(ALLOCATION_HANDLE, true, 0x1000); EXPECT_EQ(1u, mockTemporaryResources->makeResidentResult.called); } TEST_F(WddmLockWithMakeResidentTests, givenAllocationWhenApplyBlockingMakeResidentThenAcquireUniqueLock) { - wddm->temporaryResources->makeResidentResource(ALLOCATION_HANDLE); + wddm->temporaryResources->makeResidentResource(ALLOCATION_HANDLE, 0x1000); EXPECT_EQ(1u, mockTemporaryResources->acquireLockResult.called); EXPECT_EQ(reinterpret_cast(&mockTemporaryResources->resourcesLock), mockTemporaryResources->acquireLockResult.uint64ParamPassed); } TEST_F(WddmLockWithMakeResidentTests, givenAllocationWhenApplyBlockingMakeResidentThenCallMakeResidentAndStoreAllocation) { - wddm->temporaryResources->makeResidentResource(ALLOCATION_HANDLE); + wddm->temporaryResources->makeResidentResource(ALLOCATION_HANDLE, 0x1000); EXPECT_EQ(1u, wddm->makeResidentResult.called); EXPECT_EQ(ALLOCATION_HANDLE, mockTemporaryResources->resourceHandles.back()); } TEST_F(WddmLockWithMakeResidentTests, givenAllocationWhenApplyBlockingMakeResidentThenWaitForCurrentPagingFenceValue) { wddm->mockPagingFence = 0u; wddm->currentPagingFenceValue = 3u; - wddm->temporaryResources->makeResidentResource(ALLOCATION_HANDLE); + wddm->temporaryResources->makeResidentResource(ALLOCATION_HANDLE, 0x1000); EXPECT_EQ(1u, wddm->makeResidentResult.called); EXPECT_EQ(3u, wddm->mockPagingFence); EXPECT_EQ(3u, wddm->getPagingFenceAddressResult.called); @@ -923,8 +924,8 @@ TEST_F(WddmLockWithMakeResidentTests, givenAllocationWhenApplyBlockingMakeReside allocation.handle = 0x3; GmockWddm gmockWddm(*executionEnvironment->rootDeviceEnvironments[0].get()); auto mockTemporaryResources = reinterpret_cast(gmockWddm.temporaryResources.get()); - EXPECT_CALL(gmockWddm, makeResident(&allocation.handle, ::testing::_, ::testing::_, ::testing::_)).Times(2).WillRepeatedly(::testing::Return(false)); - gmockWddm.temporaryResources->makeResidentResource(allocation.handle); + EXPECT_CALL(gmockWddm, makeResident(&allocation.handle, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(2).WillRepeatedly(::testing::Return(false)); + gmockWddm.temporaryResources->makeResidentResource(allocation.handle, 0x1000); EXPECT_EQ(1u, mockTemporaryResources->evictAllResourcesResult.called); } TEST_F(WddmLockWithMakeResidentTests, whenApplyBlockingMakeResidentAndTemporaryResourcesAreEvictedSuccessfullyThenCallMakeResidentOneMoreTime) { @@ -934,8 +935,8 @@ TEST_F(WddmLockWithMakeResidentTests, whenApplyBlockingMakeResidentAndTemporaryR auto mockTemporaryResources = reinterpret_cast(gmockWddm.temporaryResources.get()); mockTemporaryResources->resourceHandles.push_back(allocation.handle); EXPECT_CALL(gmockWddm, evict(::testing::_, ::testing::_, ::testing::_)).Times(1).WillRepeatedly(::testing::Return(true)); - EXPECT_CALL(gmockWddm, makeResident(&allocation.handle, ::testing::_, ::testing::_, ::testing::_)).Times(3).WillRepeatedly(::testing::Return(false)); - gmockWddm.temporaryResources->makeResidentResource(allocation.handle); + EXPECT_CALL(gmockWddm, makeResident(&allocation.handle, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(3).WillRepeatedly(::testing::Return(false)); + gmockWddm.temporaryResources->makeResidentResource(allocation.handle, 0x1000); EXPECT_EQ(2u, mockTemporaryResources->evictAllResourcesResult.called); } TEST_F(WddmLockWithMakeResidentTests, whenApplyBlockingMakeResidentAndMakeResidentStillFailsThenDontStoreTemporaryResource) { @@ -945,9 +946,9 @@ TEST_F(WddmLockWithMakeResidentTests, whenApplyBlockingMakeResidentAndMakeReside auto mockTemporaryResources = reinterpret_cast(gmockWddm.temporaryResources.get()); mockTemporaryResources->resourceHandles.push_back(0x1); EXPECT_CALL(gmockWddm, evict(::testing::_, ::testing::_, ::testing::_)).Times(1).WillRepeatedly(::testing::Return(true)); - EXPECT_CALL(gmockWddm, makeResident(&allocation.handle, ::testing::_, ::testing::_, ::testing::_)).Times(3).WillRepeatedly(::testing::Return(false)); + EXPECT_CALL(gmockWddm, makeResident(&allocation.handle, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(3).WillRepeatedly(::testing::Return(false)); EXPECT_EQ(1u, mockTemporaryResources->resourceHandles.size()); - gmockWddm.temporaryResources->makeResidentResource(allocation.handle); + gmockWddm.temporaryResources->makeResidentResource(allocation.handle, 0x1000); EXPECT_EQ(0u, mockTemporaryResources->resourceHandles.size()); } TEST_F(WddmLockWithMakeResidentTests, whenApplyBlockingMakeResidentAndMakeResidentPassesAfterEvictThenStoreTemporaryResource) { @@ -957,9 +958,9 @@ TEST_F(WddmLockWithMakeResidentTests, whenApplyBlockingMakeResidentAndMakeReside auto mockTemporaryResources = reinterpret_cast(gmockWddm.temporaryResources.get()); mockTemporaryResources->resourceHandles.push_back(0x1); EXPECT_CALL(gmockWddm, evict(::testing::_, ::testing::_, ::testing::_)).Times(1).WillRepeatedly(::testing::Return(true)); - EXPECT_CALL(gmockWddm, makeResident(&allocation.handle, ::testing::_, ::testing::_, ::testing::_)).Times(2).WillOnce(::testing::Return(false)).WillOnce(::testing::Return(true)); + EXPECT_CALL(gmockWddm, makeResident(&allocation.handle, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(2).WillOnce(::testing::Return(false)).WillOnce(::testing::Return(true)); EXPECT_EQ(1u, mockTemporaryResources->resourceHandles.size()); - gmockWddm.temporaryResources->makeResidentResource(allocation.handle); + gmockWddm.temporaryResources->makeResidentResource(allocation.handle, 0x1000); EXPECT_EQ(1u, mockTemporaryResources->resourceHandles.size()); EXPECT_EQ(0x2, mockTemporaryResources->resourceHandles.back()); } @@ -969,8 +970,8 @@ TEST_F(WddmLockWithMakeResidentTests, whenApplyBlockingMakeResidentAndMakeReside GmockWddm gmockWddm(*executionEnvironment->rootDeviceEnvironments[0].get()); auto mockTemporaryResources = reinterpret_cast(gmockWddm.temporaryResources.get()); mockTemporaryResources->resourceHandles.push_back(0x1); - EXPECT_CALL(gmockWddm, makeResident(&allocation.handle, ::testing::_, ::testing::_, ::testing::_)).Times(1).WillOnce(::testing::Return(true)); - gmockWddm.temporaryResources->makeResidentResource(allocation.handle); + EXPECT_CALL(gmockWddm, makeResident(&allocation.handle, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(1).WillOnce(::testing::Return(true)); + gmockWddm.temporaryResources->makeResidentResource(allocation.handle, 0x1000); EXPECT_EQ(2u, mockTemporaryResources->resourceHandles.size()); EXPECT_EQ(0x2, mockTemporaryResources->resourceHandles.back()); } @@ -1266,3 +1267,129 @@ TEST(HwDeviceId, whenHwDeviceIdIsDestroyedThenAdapterIsClosed) { EXPECT_EQ(1u, GdiWithMockedCloseFunc::closeAdapterCalled); EXPECT_EQ(adapter, GdiWithMockedCloseFunc::closeAdapterCalledArgPassed); } + +namespace NEO { +namespace ResLog { +extern uint32_t mockFopenCalled; +extern uint32_t mockVfptrinfCalled; +extern uint32_t mockFcloseCalled; +} // namespace ResLog +} // namespace NEO + +TEST_F(WddmTest, WhenResidencyLoggingEnabledThenExpectLoggerCreated) { + NEO::ResLog::mockFopenCalled = 0; + NEO::ResLog::mockVfptrinfCalled = 0; + NEO::ResLog::mockFcloseCalled = 0; + DebugManagerStateRestore dbgRestore; + DebugManager.flags.WddmResidencyLogger.set(true); + + wddm->createPagingFenceLogger(); + EXPECT_NE(nullptr, wddm->residencyLogger.get()); + wddm->residencyLogger.reset(); + if (NEO::residencyLoggingAvailable) { + EXPECT_EQ(1u, NEO::ResLog::mockFopenCalled); + EXPECT_EQ(1u, NEO::ResLog::mockVfptrinfCalled); + EXPECT_EQ(1u, NEO::ResLog::mockFcloseCalled); + } +} + +TEST_F(WddmTest, GivenResidencyLoggingEnabledWhenMakeResidentSuccessThenExpectSizeRapport) { + if (!NEO::residencyLoggingAvailable) { + GTEST_SKIP(); + } + NEO::ResLog::mockFopenCalled = 0; + NEO::ResLog::mockVfptrinfCalled = 0; + NEO::ResLog::mockFcloseCalled = 0; + + DebugManagerStateRestore dbgRestore; + DebugManager.flags.WddmResidencyLogger.set(true); + wddm->callBaseCreatePagingLogger = false; + + wddm->createPagingFenceLogger(); + EXPECT_NE(nullptr, wddm->residencyLogger.get()); + auto logger = static_cast(wddm->residencyLogger.get()); + + D3DKMT_HANDLE handle = 0x10; + uint64_t bytesToTrim = 0; + wddm->makeResident(&handle, 1, false, &bytesToTrim, 0x1000); + + //2 - one for open log, second for allocation size + EXPECT_EQ(2u, NEO::ResLog::mockVfptrinfCalled); + EXPECT_TRUE(logger->makeResidentCall); +} + +TEST_F(WddmTest, GivenResidencyLoggingEnabledWhenMakeResidentFailThenExpectTrimReport) { + if (!NEO::residencyLoggingAvailable) { + GTEST_SKIP(); + } + NEO::ResLog::mockFopenCalled = 0; + NEO::ResLog::mockVfptrinfCalled = 0; + NEO::ResLog::mockFcloseCalled = 0; + + DebugManagerStateRestore dbgRestore; + DebugManager.flags.WddmResidencyLogger.set(true); + wddm->callBaseCreatePagingLogger = false; + + wddm->createPagingFenceLogger(); + EXPECT_NE(nullptr, wddm->residencyLogger.get()); + auto logger = static_cast(wddm->residencyLogger.get()); + + D3DKMT_HANDLE handle = static_cast(-1); + uint64_t bytesToTrim = 0; + + wddm->makeResident(&handle, 1, false, &bytesToTrim, 0x1000); + + //3 - one for open log, second for report allocations, 3rd for trim size + EXPECT_EQ(3u, NEO::ResLog::mockVfptrinfCalled); + EXPECT_FALSE(logger->makeResidentCall); +} + +TEST_F(WddmTest, GivenResidencyLoggingEnabledWhenEnterWaitCalledThenExpectInternalFlagOn) { + if (!NEO::residencyLoggingAvailable) { + GTEST_SKIP(); + } + NEO::ResLog::mockFopenCalled = 0; + NEO::ResLog::mockVfptrinfCalled = 0; + NEO::ResLog::mockFcloseCalled = 0; + + DebugManagerStateRestore dbgRestore; + DebugManager.flags.WddmResidencyLogger.set(true); + wddm->callBaseCreatePagingLogger = false; + + wddm->createPagingFenceLogger(); + EXPECT_NE(nullptr, wddm->residencyLogger.get()); + auto logger = static_cast(wddm->residencyLogger.get()); + logger->enteredWait(); + EXPECT_TRUE(logger->enterWait); +} + +TEST_F(WddmTest, GivenResidencyLoggingEnabledWhenMakeResidentAndWaitPagingThenExpectFlagsOff) { + if (!NEO::residencyLoggingAvailable) { + GTEST_SKIP(); + } + NEO::ResLog::mockFopenCalled = 0; + NEO::ResLog::mockVfptrinfCalled = 0; + NEO::ResLog::mockFcloseCalled = 0; + + DebugManagerStateRestore dbgRestore; + DebugManager.flags.WddmResidencyLogger.set(true); + wddm->callBaseCreatePagingLogger = false; + + wddm->createPagingFenceLogger(); + EXPECT_NE(nullptr, wddm->residencyLogger.get()); + auto logger = static_cast(wddm->residencyLogger.get()); + + D3DKMT_HANDLE handle = 0x10; + uint64_t bytesToTrim = 0; + wddm->makeResident(&handle, 1, false, &bytesToTrim, 0x1000); + + //2 - one for open log, second for allocation size + EXPECT_EQ(2u, NEO::ResLog::mockVfptrinfCalled); + EXPECT_TRUE(logger->makeResidentCall); + + logger->enterWait = true; + wddm->waitOnPagingFenceFromCpu(); + EXPECT_EQ(4u, NEO::ResLog::mockVfptrinfCalled); + EXPECT_FALSE(logger->makeResidentCall); + EXPECT_FALSE(logger->enterWait); +} diff --git a/opencl/test/unit_test/os_interface/windows/wddm_create.cpp b/opencl/test/unit_test/os_interface/windows/wddm_create.cpp index 069ed4446e..8df845ae6a 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_create.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_create.cpp @@ -6,9 +6,20 @@ */ #include "opencl/test/unit_test/mocks/mock_wddm.h" +#include "opencl/test/unit_test/mocks/mock_wddm_residency_logger_functions.h" namespace NEO { Wddm *Wddm::createWddm(std::unique_ptr hwDeviceId, RootDeviceEnvironment &rootDeviceEnvironment) { return new WddmMock(rootDeviceEnvironment); } + +namespace ResLog { +fopenFuncPtr fopenPtr = &mockFopen; +vfprintfFuncPtr vfprintfPtr = &mockVfptrinf; +fcloseFuncPtr fclosePtr = &mockFclose; + +uint32_t mockFopenCalled = 0; +uint32_t mockVfptrinfCalled = 0; +uint32_t mockFcloseCalled = 0; +} // namespace ResLog } // namespace NEO diff --git a/opencl/test/unit_test/os_interface/windows/wddm_kmdaf_listener_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_kmdaf_listener_tests.cpp index 9480fbb137..42fb952aac 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_kmdaf_listener_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_kmdaf_listener_tests.cpp @@ -54,7 +54,7 @@ class WddmKmDafListenerTest : public ::testing::Test { }; TEST_F(WddmKmDafListenerTest, givenWddmWhenLockResourceIsCalledThenKmDafListenerNotifyLockIsFedWithCorrectParams) { - wddmWithKmDafMock->lockResource(ALLOCATION_HANDLE, false); + wddmWithKmDafMock->lockResource(ALLOCATION_HANDLE, false, 0x1000); EXPECT_EQ(wddmWithKmDafMock->featureTable->ftrKmdDaf, wddmWithKmDafMock->getKmDafListenerMock().notifyLockParametrization.ftrKmdDaf); EXPECT_EQ(wddmWithKmDafMock->getAdapter(), wddmWithKmDafMock->getKmDafListenerMock().notifyLockParametrization.hAdapter); @@ -105,7 +105,7 @@ TEST_F(WddmKmDafListenerTest, givenWddmWhenFreeGpuVirtualAddressIsCalledThenKmDa TEST_F(WddmKmDafListenerTest, givenWddmWhenMakeResidentIsCalledThenKmDafListenerNotifyMakeResidentIsFedWithCorrectParams) { MockWddmAllocation allocation; - wddmWithKmDafMock->makeResident(&allocation.handle, 1, false, nullptr); + wddmWithKmDafMock->makeResident(&allocation.handle, 1, false, nullptr, 0x1000); EXPECT_EQ(wddmWithKmDafMock->featureTable->ftrKmdDaf, wddmWithKmDafMock->getKmDafListenerMock().notifyMakeResidentParametrization.ftrKmdDaf); EXPECT_EQ(wddmWithKmDafMock->getAdapter(), wddmWithKmDafMock->getKmDafListenerMock().notifyMakeResidentParametrization.hAdapter); diff --git a/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp index 9660c4ed8f..287e0551c6 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp @@ -962,10 +962,10 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, makeResidentResidency TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallingMakeResidentResidencyAllocationsThenDontMarkAllocationsAsResident) { MockWddmAllocation allocation1, allocation2, allocation3, allocation4; - auto makeResidentWithOutBytesToTrim = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) -> bool { *numberOfBytesToTrim = 4 * 4096; return false; }; + auto makeResidentWithOutBytesToTrim = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t size) -> bool { *numberOfBytesToTrim = 4 * 4096; return false; }; - ON_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_)).WillByDefault(::testing::Invoke(makeResidentWithOutBytesToTrim)); - EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(2); + ON_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).WillByDefault(::testing::Invoke(makeResidentWithOutBytesToTrim)); + EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(2); ResidencyContainer residencyPack{&allocation1, &allocation2, &allocation3, &allocation4}; bool result = residencyController->makeResidentResidencyAllocations(residencyPack); @@ -984,10 +984,10 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin WddmAllocation *allocationTriple = static_cast(memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{false, 2 * MemoryConstants::pageSize}, ptr)); ASSERT_NE(nullptr, allocationTriple); - auto makeResidentWithOutBytesToTrim = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) -> bool { *numberOfBytesToTrim = 4 * 4096; return false; }; + auto makeResidentWithOutBytesToTrim = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t size) -> bool { *numberOfBytesToTrim = 4 * 4096; return false; }; - ON_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_)).WillByDefault(::testing::Invoke(makeResidentWithOutBytesToTrim)); - EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(2); + ON_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).WillByDefault(::testing::Invoke(makeResidentWithOutBytesToTrim)); + EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(2); ResidencyContainer residencyPack{&allocation1, allocationTriple, &allocation2}; bool result = residencyController->makeResidentResidencyAllocations(residencyPack); @@ -1004,11 +1004,11 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallingMakeResidentResidencyAllocationsThenCallItAgainWithCantTrimFurtherSetToTrue) { MockWddmAllocation allocation1; - auto makeResidentWithOutBytesToTrim = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) -> bool { *numberOfBytesToTrim = 4 * 4096; return false; }; + auto makeResidentWithOutBytesToTrim = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t size) -> bool { *numberOfBytesToTrim = 4 * 4096; return false; }; - ON_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_)).WillByDefault(::testing::Invoke(makeResidentWithOutBytesToTrim)); - EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, false, ::testing::_)).Times(1); - EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, true, ::testing::_)).Times(1); + ON_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).WillByDefault(::testing::Invoke(makeResidentWithOutBytesToTrim)); + EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, false, ::testing::_, ::testing::_)).Times(1); + EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, true, ::testing::_, ::testing::_)).Times(1); ResidencyContainer residencyPack{&allocation1}; bool result = residencyController->makeResidentResidencyAllocations(residencyPack); @@ -1023,13 +1023,13 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenAllocationPackPassedWhenCal allocation2.handle = 2; ResidencyContainer residencyPack{&allocation1, &allocation2}; - auto makeResidentWithOutBytesToTrim = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) -> bool { + auto makeResidentWithOutBytesToTrim = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t size) -> bool { EXPECT_EQ(1, handles[0]); EXPECT_EQ(2, handles[1]); return true; }; - ON_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_)).WillByDefault(::testing::Invoke(makeResidentWithOutBytesToTrim)); - EXPECT_CALL(*wddm, makeResident(::testing::_, 2, false, ::testing::_)).Times(1); + ON_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).WillByDefault(::testing::Invoke(makeResidentWithOutBytesToTrim)); + EXPECT_CALL(*wddm, makeResident(::testing::_, 2, false, ::testing::_, ::testing::_)).Times(1); bool result = residencyController->makeResidentResidencyAllocations(residencyPack); EXPECT_TRUE(result); @@ -1043,9 +1043,9 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsAndTrimToB allocationToTrim.getResidencyData().updateCompletionData(residencyController->getMonitoredFence().lastSubmittedFence, osContext->getContextId()); - auto makeResidentWithOutBytesToTrim = [allocationSize](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) -> bool { *numberOfBytesToTrim = allocationSize; return false; }; + auto makeResidentWithOutBytesToTrim = [allocationSize](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t size) -> bool { *numberOfBytesToTrim = allocationSize; return false; }; - EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(2).WillOnce(::testing::Invoke(makeResidentWithOutBytesToTrim)).WillOnce(::testing::Return(true)); + EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(2).WillOnce(::testing::Invoke(makeResidentWithOutBytesToTrim)).WillOnce(::testing::Return(true)); residencyController->addToTrimCandidateList(&allocationToTrim); @@ -1061,10 +1061,10 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin MockWddmAllocation allocation1; ResidencyContainer residencyPack{&allocation1}; - auto makeResidentThatFails = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) -> bool { return false; }; - auto makeResidentThatSucceds = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) -> bool { return true; }; + auto makeResidentThatFails = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t size) -> bool { return false; }; + auto makeResidentThatSucceds = [](const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t size) -> bool { return true; }; - EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(2).WillOnce(::testing::Invoke(makeResidentThatFails)).WillOnce(::testing::Invoke(makeResidentThatSucceds)); + EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(2).WillOnce(::testing::Invoke(makeResidentThatFails)).WillOnce(::testing::Invoke(makeResidentThatSucceds)); residencyController->makeResidentResidencyAllocations(residencyPack); EXPECT_TRUE(residencyController->isMemoryBudgetExhausted()); diff --git a/opencl/test/unit_test/test_files/igdrcl.config b/opencl/test/unit_test/test_files/igdrcl.config index 19910b0eaf..fe124abcd4 100644 --- a/opencl/test/unit_test/test_files/igdrcl.config +++ b/opencl/test/unit_test/test_files/igdrcl.config @@ -133,4 +133,5 @@ MakeAllBuffersResident = 0 EnableDirectSubmission = -1 DirectSubmissionBufferPlacement = -1 DirectSubmissionSemaphorePlacement = -1 -DirectSubmissionDisableCpuCacheFlush = -1 \ No newline at end of file +DirectSubmissionDisableCpuCacheFlush = -1 +WddmResidencyLogger = 0 \ No newline at end of file diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index 248249cdc4..ee0eb467da 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -79,6 +79,7 @@ DECLARE_DEBUG_VARIABLE(bool, PrintEMDebugInformation, false, "prints execution m DECLARE_DEBUG_VARIABLE(bool, PrintLWSSizes, false, "prints driver choosen local workgroup sizes") DECLARE_DEBUG_VARIABLE(bool, PrintDispatchParameters, false, "prints dispatch paramters of kernels passed to clEnqueueNDRangeKernel") DECLARE_DEBUG_VARIABLE(bool, PrintProgramBinaryProcessingTime, false, "prints execution time of Program::processGenBinary() method during program building") +DECLARE_DEBUG_VARIABLE(bool, WddmResidencyLogger, false, "gather Wddm residency statistics to file") DECLARE_DEBUG_VARIABLE(int32_t, PrintDriverDiagnostics, -1, "prints driver diagnostics messages to standard output, value corresponds to hint level") /*PERFORMANCE FLAGS*/ DECLARE_DEBUG_VARIABLE(bool, EnableNullHardware, false, "works on Windows only, sets the Null Hardware flag that makes all Command buffers completed while GPU does nothing") diff --git a/shared/source/os_interface/windows/CMakeLists.txt b/shared/source/os_interface/windows/CMakeLists.txt index 6dd4f83742..8597cb6d12 100644 --- a/shared/source/os_interface/windows/CMakeLists.txt +++ b/shared/source/os_interface/windows/CMakeLists.txt @@ -50,6 +50,8 @@ set(NEO_CORE_OS_INTERFACE_WINDOWS ${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.h ${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_defs.h + ${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_residency_logger.h + ${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_residency_logger_defs.h ${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.h ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_memory_manager_allocate_in_device_pool.cpp diff --git a/shared/source/os_interface/windows/wddm/wddm.cpp b/shared/source/os_interface/windows/wddm/wddm.cpp index f96cdda36f..09c359eccb 100644 --- a/shared/source/os_interface/windows/wddm/wddm.cpp +++ b/shared/source/os_interface/windows/wddm/wddm.cpp @@ -23,6 +23,7 @@ #include "shared/source/os_interface/windows/os_interface.h" #include "shared/source/os_interface/windows/sys_calls.h" #include "shared/source/os_interface/windows/wddm/wddm_interface.h" +#include "shared/source/os_interface/windows/wddm/wddm_residency_logger.h" #include "shared/source/os_interface/windows/wddm_allocation.h" #include "shared/source/os_interface/windows/wddm_engine_mapper.h" #include "shared/source/os_interface/windows/wddm_residency_allocations_container.h" @@ -177,6 +178,7 @@ bool Wddm::createPagingQueue() { pagingQueue = CreatePagingQueue.hPagingQueue; pagingQueueSyncObject = CreatePagingQueue.hSyncObject; pagingFenceAddress = reinterpret_cast(CreatePagingQueue.FenceValueCPUVirtualAddress); + createPagingFenceLogger(); } return status == STATUS_SUCCESS; @@ -335,12 +337,14 @@ bool Wddm::evict(const D3DKMT_HANDLE *handleList, uint32_t numOfHandles, uint64_ return status == STATUS_SUCCESS; } -bool Wddm::makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) { +bool Wddm::makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t totalSize) { NTSTATUS status = STATUS_SUCCESS; D3DDDI_MAKERESIDENT makeResident = {0}; UINT priority = 0; bool success = false; + perfLogResidencyReportAllocations(residencyLogger.get(), count, totalSize); + makeResident.AllocationList = handles; makeResident.hPagingQueue = pagingQueue; makeResident.NumAllocations = count; @@ -349,14 +353,16 @@ bool Wddm::makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantT makeResident.Flags.MustSucceed = cantTrimFurther ? 1 : 0; status = getGdi()->makeResident(&makeResident); - if (status == STATUS_PENDING) { + perfLogResidencyMakeResident(residencyLogger.get(), true); updatePagingFenceValue(makeResident.PagingFenceValue); success = true; } else if (status == STATUS_SUCCESS) { + perfLogResidencyMakeResident(residencyLogger.get(), false); success = true; } else { DEBUG_BREAK_IF(true); + perfLogResidencyTrimRequired(residencyLogger.get(), makeResident.NumBytesToTrim); if (numberOfBytesToTrim != nullptr) *numberOfBytesToTrim = makeResident.NumBytesToTrim; UNRECOVERABLE_IF(cantTrimFurther); @@ -683,10 +689,10 @@ bool Wddm::openNTHandle(HANDLE handle, WddmAllocation *alloc) { return true; } -void *Wddm::lockResource(const D3DKMT_HANDLE &handle, bool applyMakeResidentPriorToLock) { +void *Wddm::lockResource(const D3DKMT_HANDLE &handle, bool applyMakeResidentPriorToLock, size_t size) { if (applyMakeResidentPriorToLock) { - temporaryResources->makeResidentResource(handle); + temporaryResources->makeResidentResource(handle, size); } NTSTATUS status = STATUS_UNSUCCESSFUL; @@ -1001,8 +1007,11 @@ bool Wddm::configureDeviceAddressSpace() { } void Wddm::waitOnPagingFenceFromCpu() { + perfLogStartWaitTime(residencyLogger.get()); while (currentPagingFenceValue > *getPagingFenceAddress()) - ; + perfLogResidencyEnteredWait(residencyLogger.get()); + + perfLogResidencyWaitPagingeFenceLog(residencyLogger.get()); } void Wddm::setGmmInputArg(void *args) { @@ -1021,4 +1030,10 @@ WddmVersion Wddm::getWddmVersion() { } } +void Wddm::createPagingFenceLogger() { + if (DebugManager.flags.WddmResidencyLogger.get()) { + residencyLogger = std::make_unique(device, pagingFenceAddress); + } +} + } // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/wddm.h b/shared/source/os_interface/windows/wddm/wddm.h index 279676e956..b64f2804af 100644 --- a/shared/source/os_interface/windows/wddm/wddm.h +++ b/shared/source/os_interface/windows/wddm/wddm.h @@ -13,6 +13,7 @@ #include "shared/source/os_interface/os_context.h" #include "shared/source/os_interface/windows/hw_device_id.h" #include "shared/source/os_interface/windows/wddm/wddm_defs.h" +#include "shared/source/os_interface/windows/wddm/wddm_residency_logger.h" #include "shared/source/utilities/spinlock.h" #include "sku_info.h" @@ -54,7 +55,7 @@ class Wddm { bool init(); MOCKABLE_VIRTUAL bool evict(const D3DKMT_HANDLE *handleList, uint32_t numOfHandles, uint64_t &sizeToTrim); - MOCKABLE_VIRTUAL bool makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim); + MOCKABLE_VIRTUAL bool makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t totalSize); MOCKABLE_VIRTUAL bool mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_VIRTUAL_ADDRESS preferredAddress, D3DGPU_VIRTUAL_ADDRESS &gpuPtr); bool mapGpuVirtualAddress(AllocationStorageData *allocationStorageData); MOCKABLE_VIRTUAL D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size); @@ -67,7 +68,7 @@ class Wddm { MOCKABLE_VIRTUAL bool destroyAllocations(const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle); MOCKABLE_VIRTUAL bool openSharedHandle(D3DKMT_HANDLE handle, WddmAllocation *alloc); bool openNTHandle(HANDLE handle, WddmAllocation *alloc); - MOCKABLE_VIRTUAL void *lockResource(const D3DKMT_HANDLE &handle, bool applyMakeResidentPriorToLock); + MOCKABLE_VIRTUAL void *lockResource(const D3DKMT_HANDLE &handle, bool applyMakeResidentPriorToLock, size_t size); MOCKABLE_VIRTUAL void unlockResource(const D3DKMT_HANDLE &handle); MOCKABLE_VIRTUAL void kmDafLock(D3DKMT_HANDLE handle); MOCKABLE_VIRTUAL bool isKmDafEnabled() const { return featureTable->ftrKmdDaf; } @@ -184,6 +185,7 @@ class Wddm { bool destroyDevice(); void getDeviceState(); void handleCompletion(OsContextWin &osContext); + MOCKABLE_VIRTUAL void createPagingFenceLogger(); static GetSystemInfoFcn getSystemInfo; static VirtualFreeFcn virtualFreeFnc; @@ -192,5 +194,6 @@ class Wddm { std::unique_ptr kmDafListener; std::unique_ptr wddmInterface; std::unique_ptr temporaryResources; + std::unique_ptr residencyLogger; }; } // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/wddm_create.cpp b/shared/source/os_interface/windows/wddm/wddm_create.cpp index 3e8ce8a856..5dfdee7bf3 100644 --- a/shared/source/os_interface/windows/wddm/wddm_create.cpp +++ b/shared/source/os_interface/windows/wddm/wddm_create.cpp @@ -11,4 +11,11 @@ namespace NEO { Wddm *Wddm::createWddm(std::unique_ptr hwDeviceId, RootDeviceEnvironment &rootDeviceEnvironment) { return new Wddm(std::move(hwDeviceId), rootDeviceEnvironment); } + +namespace ResLog { +fopenFuncPtr fopenPtr = &fopen; +vfprintfFuncPtr vfprintfPtr = &vfprintf; +fcloseFuncPtr fclosePtr = &fclose; +} // namespace ResLog + } // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/wddm_residency_logger.h b/shared/source/os_interface/windows/wddm/wddm_residency_logger.h new file mode 100644 index 0000000000..7da02d89e7 --- /dev/null +++ b/shared/source/os_interface/windows/wddm/wddm_residency_logger.h @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include "shared/source/helpers/debug_helpers.h" +#include "shared/source/os_interface/windows/wddm/wddm_defs.h" +#include "shared/source/os_interface/windows/wddm/wddm_residency_logger_defs.h" + +#include +#include + +namespace NEO { + +class WddmResidencyLogger { + public: + WddmResidencyLogger(D3DKMT_HANDLE device, VOID *fenceValueCpuVirtualAddress) { + std::stringstream id; + id << std::hex; + id << "device-0x" << device << "_" + << "pfencecpu-0x" << fenceValueCpuVirtualAddress; + std::stringstream filename; + filename << "pagingfence_" << id.str() << ".log"; + pagingLog = ResLog::fopenPtr(filename.str().c_str(), "at"); + UNRECOVERABLE_IF(pagingLog == nullptr); + fPagingLog("%s\n", id.str().c_str()); + } + + ~WddmResidencyLogger() { + ResLog::fclosePtr(pagingLog); + } + + void reportAllocations(uint32_t count, size_t size) { + fPagingLog("residency for: handles %u size %zu\n", count, size); + } + + void makeResidentLog(bool pendingMakeResident) { + this->pendingMakeResident = pendingMakeResident; + makeResidentCall = true; + pendingTime = std::chrono::high_resolution_clock::now(); + } + + void startWaitTime() { + waitStartTime = std::chrono::high_resolution_clock::now(); + } + + void enteredWait() { + enterWait = true; + } + + void waitPagingeFenceLog() { + endTime = std::chrono::high_resolution_clock::now(); + + int64_t timeDiff = 0; + + timeDiff = std::chrono::duration_cast(endTime - pendingTime).count(); + fPagingLog("makeResidentCall: %x pending return: %x delta time makeResident: %lld\n", + makeResidentCall, + pendingMakeResident, + timeDiff); + + timeDiff = std::chrono::duration_cast(endTime - waitStartTime).count(); + fPagingLog("waiting: %x delta time wait loop: %lld\n", enterWait, timeDiff); + + makeResidentCall = false; + enterWait = false; + } + + void trimRequired(UINT64 numBytesToTrim) { + fPagingLog("trimming required: bytes to trim: %llu\n", numBytesToTrim); + } + + protected: + void fPagingLog(char const *const formatStr, ...) { + va_list args; + va_start(args, formatStr); + ResLog::vfprintfPtr(pagingLog, formatStr, args); + va_end(args); + } + + bool pendingMakeResident = false; + bool enterWait = false; + bool makeResidentCall = false; + FILE *pagingLog = nullptr; + std::chrono::high_resolution_clock::time_point pendingTime; + std::chrono::high_resolution_clock::time_point waitStartTime; + std::chrono::high_resolution_clock::time_point endTime; +}; + +inline void perfLogResidencyMakeResident(WddmResidencyLogger *log, bool pendingMakeResident) { + if (residencyLoggingAvailable) { + if (log) { + log->makeResidentLog(pendingMakeResident); + } + } +} + +inline void perfLogResidencyReportAllocations(WddmResidencyLogger *log, uint32_t count, size_t size) { + if (residencyLoggingAvailable) { + if (log) { + log->reportAllocations(count, size); + } + } +} + +inline void perfLogStartWaitTime(WddmResidencyLogger *log) { + if (residencyLoggingAvailable) { + if (log) { + log->startWaitTime(); + } + } +} + +inline void perfLogResidencyEnteredWait(WddmResidencyLogger *log) { + if (residencyLoggingAvailable) { + if (log) { + log->enteredWait(); + } + } +} + +inline void perfLogResidencyWaitPagingeFenceLog(WddmResidencyLogger *log) { + if (residencyLoggingAvailable) { + if (log) { + log->waitPagingeFenceLog(); + } + } +} + +inline void perfLogResidencyTrimRequired(WddmResidencyLogger *log, UINT64 numBytesToTrim) { + if (residencyLoggingAvailable) { + if (log) { + log->trimRequired(numBytesToTrim); + } + } +} + +} // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/wddm_residency_logger_defs.h b/shared/source/os_interface/windows/wddm/wddm_residency_logger_defs.h new file mode 100644 index 0000000000..ba98cdcd38 --- /dev/null +++ b/shared/source/os_interface/windows/wddm/wddm_residency_logger_defs.h @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include +#include + +namespace NEO { +namespace ResLog { +using fopenFuncPtr = FILE *(*)(const char *, const char *); +using vfprintfFuncPtr = int (*)(FILE *, char const *const formatStr, va_list arg); +using fcloseFuncPtr = int (*)(FILE *); + +extern fopenFuncPtr fopenPtr; +extern vfprintfFuncPtr vfprintfPtr; +extern fcloseFuncPtr fclosePtr; +} // namespace ResLog + +#if defined(_RELEASE_INTERNAL) || (_DEBUG) +constexpr bool residencyLoggingAvailable = true; +#else +constexpr bool residencyLoggingAvailable = false; +#endif +} // namespace NEO diff --git a/shared/source/os_interface/windows/wddm_memory_manager.cpp b/shared/source/os_interface/windows/wddm_memory_manager.cpp index be5eaa22e3..06e0195d36 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.cpp +++ b/shared/source/os_interface/windows/wddm_memory_manager.cpp @@ -298,7 +298,7 @@ void WddmMemoryManager::removeAllocationFromHostPtrManager(GraphicsAllocation *g void *WddmMemoryManager::lockResourceImpl(GraphicsAllocation &graphicsAllocation) { auto &wddmAllocation = static_cast(graphicsAllocation); - return getWddm(graphicsAllocation.getRootDeviceIndex()).lockResource(wddmAllocation.getDefaultHandle(), wddmAllocation.needsMakeResidentBeforeLock); + return getWddm(graphicsAllocation.getRootDeviceIndex()).lockResource(wddmAllocation.getDefaultHandle(), wddmAllocation.needsMakeResidentBeforeLock, wddmAllocation.getAlignedSize()); } void WddmMemoryManager::unlockResourceImpl(GraphicsAllocation &graphicsAllocation) { auto &wddmAllocation = static_cast(graphicsAllocation); diff --git a/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp b/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp index 875212ba1c..a100121d8f 100644 --- a/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp +++ b/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp @@ -23,9 +23,11 @@ MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(ArrayRef handlesForResidency; + size_t totalSize = 0; for (const auto &allocation : gfxAllocations) { WddmAllocation *wddmAllocation = reinterpret_cast(allocation); + totalSize += wddmAllocation->getAlignedSize(); if (wddmAllocation->fragmentsStorage.fragmentCount > 0) { for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { @@ -39,7 +41,7 @@ MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(ArrayRefgetNumHandles(); } } - return residentAllocations->makeResidentResources(handlesForResidency.begin(), totalHandlesCount); + return residentAllocations->makeResidentResources(handlesForResidency.begin(), totalHandlesCount, totalSize); } MemoryOperationsStatus WddmMemoryOperationsHandler::evict(GraphicsAllocation &gfxAllocation) { diff --git a/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp b/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp index 627f795e55..0124587e35 100644 --- a/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp +++ b/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp @@ -56,17 +56,17 @@ MemoryOperationsStatus WddmResidentAllocationsContainer::evictResources(const D3 return MemoryOperationsStatus::SUCCESS; } -MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResource(const D3DKMT_HANDLE &handle) { - return makeResidentResources(&handle, 1u); +MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResource(const D3DKMT_HANDLE &handle, size_t size) { + return makeResidentResources(&handle, 1u, size); } -MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count) { +MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count, size_t size) { bool madeResident = false; - while (!(madeResident = wddm->makeResident(handles, count, false, nullptr))) { + while (!(madeResident = wddm->makeResident(handles, count, false, nullptr, size))) { if (evictAllResources() == MemoryOperationsStatus::SUCCESS) { continue; } - if (!wddm->makeResident(handles, count, false, nullptr)) { + if (!wddm->makeResident(handles, count, false, nullptr, size)) { DEBUG_BREAK_IF(true); return MemoryOperationsStatus::OUT_OF_MEMORY; }; diff --git a/shared/source/os_interface/windows/wddm_residency_allocations_container.h b/shared/source/os_interface/windows/wddm_residency_allocations_container.h index a7f2883adc..628332daf1 100644 --- a/shared/source/os_interface/windows/wddm_residency_allocations_container.h +++ b/shared/source/os_interface/windows/wddm_residency_allocations_container.h @@ -25,8 +25,8 @@ class WddmResidentAllocationsContainer { MOCKABLE_VIRTUAL MemoryOperationsStatus evictAllResources(); MOCKABLE_VIRTUAL MemoryOperationsStatus evictResource(const D3DKMT_HANDLE &handle); MemoryOperationsStatus evictResources(const D3DKMT_HANDLE *handles, const uint32_t count); - MOCKABLE_VIRTUAL MemoryOperationsStatus makeResidentResource(const D3DKMT_HANDLE &handle); - MemoryOperationsStatus makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count); + MOCKABLE_VIRTUAL MemoryOperationsStatus makeResidentResource(const D3DKMT_HANDLE &handle, size_t size); + MemoryOperationsStatus makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count, size_t size); MOCKABLE_VIRTUAL void removeResource(const D3DKMT_HANDLE &handle); protected: diff --git a/shared/source/os_interface/windows/wddm_residency_controller.cpp b/shared/source/os_interface/windows/wddm_residency_controller.cpp index f2ad3d2967..2ab4b7397b 100644 --- a/shared/source/os_interface/windows/wddm_residency_controller.cpp +++ b/shared/source/os_interface/windows/wddm_residency_controller.cpp @@ -301,6 +301,7 @@ bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyCo const size_t residencyCount = allocationsForResidency.size(); std::unique_ptr handlesForResidency(new D3DKMT_HANDLE[residencyCount * maxFragmentsCount * EngineLimits::maxHandleCount]); uint32_t totalHandlesCount = 0; + size_t totalSize = 0; auto lock = this->acquireLock(); @@ -310,6 +311,7 @@ bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyCo WddmAllocation *allocation = static_cast(allocationsForResidency[i]); ResidencyData &residencyData = allocation->getResidencyData(); bool fragmentResidency[3] = {false, false, false}; + totalSize += allocation->getAlignedSize(); DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation =", allocation, residencyData.resident[osContextId] ? "resident" : "not resident"); @@ -337,7 +339,7 @@ bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyCo bool result = true; if (totalHandlesCount) { uint64_t bytesToTrim = 0; - while ((result = wddm.makeResident(handlesForResidency.get(), totalHandlesCount, false, &bytesToTrim)) == false) { + while ((result = wddm.makeResident(handlesForResidency.get(), totalHandlesCount, false, &bytesToTrim, totalSize)) == false) { this->setMemoryBudgetExhausted(); const bool trimmingDone = this->trimResidencyToBudget(bytesToTrim); if (!trimmingDone) { @@ -346,7 +348,7 @@ bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyCo continue; } DEBUG_BREAK_IF(evictionStatus != MemoryOperationsStatus::MEMORY_NOT_FOUND); - result = wddm.makeResident(handlesForResidency.get(), totalHandlesCount, true, &bytesToTrim); + result = wddm.makeResident(handlesForResidency.get(), totalHandlesCount, true, &bytesToTrim, totalSize); break; } } diff --git a/shared/test/unit_test/os_interface/windows/mock_gdi_interface.h b/shared/test/unit_test/os_interface/windows/mock_gdi_interface.h index c9f3665621..c790c6c62e 100644 --- a/shared/test/unit_test/os_interface/windows/mock_gdi_interface.h +++ b/shared/test/unit_test/os_interface/windows/mock_gdi_interface.h @@ -25,6 +25,9 @@ class MockGdi : public Gdi { } static NTSTATUS __stdcall makeResidentMock(IN OUT D3DDDI_MAKERESIDENT *arg) { + if (arg->AllocationList[0] == static_cast(-1)) { + return STATUS_SEVERITY_ERROR; + } getMakeResidentArg() = *arg; return 0; }