From db8c2bc57efea07ba4f7a12e39bb2895b324aa09 Mon Sep 17 00:00:00 2001 From: "Jablonski, Mateusz" Date: Wed, 6 Feb 2019 13:32:49 +0100 Subject: [PATCH] Unify write memory in simulated csr when aub manager is available Change-Id: I28d0496b1b1fb973af4869e5626082142b5818dd Signed-off-by: Jablonski, Mateusz --- .../aub_command_stream_receiver_hw.h | 3 +- .../aub_command_stream_receiver_hw.inl | 31 ++++++------------- ...mand_stream_receiver_simulated_common_hw.h | 3 ++ ...nd_stream_receiver_simulated_common_hw.inl | 23 ++++++++++++++ .../command_stream_receiver_simulated_hw.h | 3 +- .../tbx_command_stream_receiver_hw.h | 3 +- .../tbx_command_stream_receiver_hw.inl | 21 ++++++------- .../command_stream/aub_file_stream_tests.cpp | 11 ++----- .../tbx_command_stream_tests.cpp | 17 ++-------- unit_tests/mocks/mock_aub_csr.h | 10 +++++- unit_tests/mocks/mock_tbx_csr.h | 10 +++++- 11 files changed, 74 insertions(+), 61 deletions(-) diff --git a/runtime/command_stream/aub_command_stream_receiver_hw.h b/runtime/command_stream/aub_command_stream_receiver_hw.h index 37fe5056cc..e94efa6322 100644 --- a/runtime/command_stream/aub_command_stream_receiver_hw.h +++ b/runtime/command_stream/aub_command_stream_receiver_hw.h @@ -28,6 +28,7 @@ class AUBCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw::AUB; using ExternalAllocationsContainer = std::vector; using BaseClass::engineIndex; + using BaseClass::getParametersForWriteMemory; using BaseClass::osContext; public: @@ -49,7 +50,7 @@ class AUBCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw(this->stream); } - MOCKABLE_VIRTUAL void writeMemory(uint64_t gpuAddress, void *cpuAddress, size_t size, uint32_t memoryBank, uint64_t entryBits, DevicesBitfield devicesBitfield); + void writeMemory(uint64_t gpuAddress, void *cpuAddress, size_t size, uint32_t memoryBank, uint64_t entryBits, DevicesBitfield devicesBitfield) override; MOCKABLE_VIRTUAL bool writeMemory(GraphicsAllocation &gfxAllocation); MOCKABLE_VIRTUAL bool writeMemory(AllocationView &allocationView); void expectMMIO(uint32_t mmioRegister, uint32_t expectedValue); diff --git a/runtime/command_stream/aub_command_stream_receiver_hw.inl b/runtime/command_stream/aub_command_stream_receiver_hw.inl index e7a46f24ba..261d69107c 100644 --- a/runtime/command_stream/aub_command_stream_receiver_hw.inl +++ b/runtime/command_stream/aub_command_stream_receiver_hw.inl @@ -13,9 +13,6 @@ #include "runtime/command_stream/aub_stream_provider.h" #include "runtime/command_stream/aub_subcapture.h" #include "runtime/execution_environment/execution_environment.h" -#include "runtime/gmm_helper/gmm.h" -#include "runtime/gmm_helper/gmm_helper.h" -#include "runtime/gmm_helper/resource_info.h" #include "runtime/helpers/aligned_memory.h" #include "runtime/helpers/debug_helpers.h" #include "runtime/helpers/hardware_context_controller.h" @@ -618,12 +615,6 @@ void AUBCommandStreamReceiverHw::makeNonResidentExternal(uint64_t gpu template void AUBCommandStreamReceiverHw::writeMemory(uint64_t gpuAddress, void *cpuAddress, size_t size, uint32_t memoryBank, uint64_t entryBits, DevicesBitfield devicesBitfield) { - if (aubManager) { - int hint = AubMemDump::DataTypeHintValues::TraceNotype; - aubManager->writeMemory(gpuAddress, cpuAddress, size, memoryBank, hint, MemoryConstants::pageSize64k); - return; - } - { std::ostringstream str; str << "ppgtt: " << std::hex << std::showbase << gpuAddress << " end address: " << gpuAddress + size << " cpu address: " << cpuAddress << " device mask: " << devicesBitfield << " size: " << std::dec << size; @@ -642,22 +633,18 @@ void AUBCommandStreamReceiverHw::writeMemory(uint64_t gpuAddress, voi template bool AUBCommandStreamReceiverHw::writeMemory(GraphicsAllocation &gfxAllocation) { - auto cpuAddress = ptrOffset(gfxAllocation.getUnderlyingBuffer(), static_cast(gfxAllocation.allocationOffset)); - auto gpuAddress = GmmHelper::decanonize(gfxAllocation.getGpuAddress()); - auto size = gfxAllocation.getUnderlyingBufferSize(); - if (gfxAllocation.gmm && gfxAllocation.gmm->isRenderCompressed) { - size = gfxAllocation.gmm->gmmResourceInfo->getSizeAllocation(); - } - - if ((size == 0) || !gfxAllocation.isAubWritable()) + uint64_t gpuAddress; + void *cpuAddress; + size_t size; + if (!this->getParametersForWriteMemory(gfxAllocation, gpuAddress, cpuAddress, size)) { return false; - - if (cpuAddress == nullptr) { - DEBUG_BREAK_IF(gfxAllocation.isLocked()); - cpuAddress = this->getMemoryManager()->lockResource(&gfxAllocation); } - writeMemory(gpuAddress, cpuAddress, size, this->getMemoryBank(&gfxAllocation), this->getPPGTTAdditionalBits(&gfxAllocation), gfxAllocation.devicesBitfield); + if (aubManager) { + this->writeMemoryWithAubManager(gfxAllocation); + } else { + writeMemory(gpuAddress, cpuAddress, size, this->getMemoryBank(&gfxAllocation), this->getPPGTTAdditionalBits(&gfxAllocation), gfxAllocation.devicesBitfield); + } if (gfxAllocation.isLocked()) { this->getMemoryManager()->unlockResource(&gfxAllocation); diff --git a/runtime/command_stream/command_stream_receiver_simulated_common_hw.h b/runtime/command_stream/command_stream_receiver_simulated_common_hw.h index 4bbbb2674d..221a1648d9 100644 --- a/runtime/command_stream/command_stream_receiver_simulated_common_hw.h +++ b/runtime/command_stream/command_stream_receiver_simulated_common_hw.h @@ -30,6 +30,7 @@ class CommandStreamReceiverSimulatedCommonHw : public CommandStreamReceiverHw hardwareContextController; diff --git a/runtime/command_stream/command_stream_receiver_simulated_common_hw.inl b/runtime/command_stream/command_stream_receiver_simulated_common_hw.inl index 1c40aaa6e9..54b6e53c0a 100644 --- a/runtime/command_stream/command_stream_receiver_simulated_common_hw.inl +++ b/runtime/command_stream/command_stream_receiver_simulated_common_hw.inl @@ -9,6 +9,10 @@ #include "runtime/aub_mem_dump/page_table_entry_bits.h" #include "runtime/command_stream/command_stream_receiver_simulated_common_hw.h" #include "runtime/helpers/hardware_context_controller.h" +#include "runtime/gmm_helper/gmm.h" +#include "runtime/gmm_helper/gmm_helper.h" +#include "runtime/gmm_helper/resource_info.h" +#include "runtime/memory_manager/memory_manager.h" #include "runtime/os_interface/debug_settings_manager.h" #include "runtime/os_interface/os_context.h" #include "third_party/aub_stream/headers/aub_manager.h" @@ -94,4 +98,23 @@ void CommandStreamReceiverSimulatedCommonHw::setupContext(OsContext & hardwareContextController = std::make_unique(*aubManager, deviceIndex, engineIndex, flags); } } + +template +bool CommandStreamReceiverSimulatedCommonHw::getParametersForWriteMemory(GraphicsAllocation &graphicsAllocation, uint64_t &gpuAddress, void *&cpuAddress, size_t &size) const { + cpuAddress = ptrOffset(graphicsAllocation.getUnderlyingBuffer(), static_cast(graphicsAllocation.allocationOffset)); + gpuAddress = GmmHelper::decanonize(graphicsAllocation.getGpuAddress()); + size = graphicsAllocation.getUnderlyingBufferSize(); + if (graphicsAllocation.gmm && graphicsAllocation.gmm->isRenderCompressed) { + size = graphicsAllocation.gmm->gmmResourceInfo->getSizeAllocation(); + } + + if ((size == 0) || !graphicsAllocation.isAubWritable()) + return false; + + if (cpuAddress == nullptr) { + cpuAddress = this->getMemoryManager()->lockResource(&graphicsAllocation); + } + return true; +} + } // namespace OCLRT diff --git a/runtime/command_stream/definitions/command_stream_receiver_simulated_hw.h b/runtime/command_stream/definitions/command_stream_receiver_simulated_hw.h index ef0c7080b9..ab363ab886 100644 --- a/runtime/command_stream/definitions/command_stream_receiver_simulated_hw.h +++ b/runtime/command_stream/definitions/command_stream_receiver_simulated_hw.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018 Intel Corporation + * Copyright (C) 2018-2019 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -28,5 +28,6 @@ class CommandStreamReceiverSimulatedHw : public CommandStreamReceiverSimulatedCo PhysicalAddressAllocator *createPhysicalAddressAllocator(const HardwareInfo *hwInfo) { return new PhysicalAddressAllocator(); } + void writeMemoryWithAubManager(GraphicsAllocation &graphicsAllocation) override{}; }; } // namespace OCLRT diff --git a/runtime/command_stream/tbx_command_stream_receiver_hw.h b/runtime/command_stream/tbx_command_stream_receiver_hw.h index a381b48652..096bbcf978 100644 --- a/runtime/command_stream/tbx_command_stream_receiver_hw.h +++ b/runtime/command_stream/tbx_command_stream_receiver_hw.h @@ -31,6 +31,7 @@ class TbxCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw BaseClass; using AUB = typename AUBFamilyMapper::AUB; using BaseClass::engineIndex; + using BaseClass::getParametersForWriteMemory; using BaseClass::osContext; uint32_t getMaskAndValueForPollForCompletion() const; @@ -48,7 +49,7 @@ class TbxCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw::pollForCompletion() { template void TbxCommandStreamReceiverHw::writeMemory(uint64_t gpuAddress, void *cpuAddress, size_t size, uint32_t memoryBank, uint64_t entryBits, DevicesBitfield devicesBitfield) { - if (aubManager) { - int hint = AubMemDump::DataTypeHintValues::TraceNotype; - aubManager->writeMemory(gpuAddress, cpuAddress, size, memoryBank, hint, MemoryConstants::pageSize64k); - return; - } AubHelperHw aubHelperHw(this->localMemoryEnabled); @@ -386,14 +381,18 @@ void TbxCommandStreamReceiverHw::writeMemory(uint64_t gpuAddress, voi template bool TbxCommandStreamReceiverHw::writeMemory(GraphicsAllocation &gfxAllocation) { - auto cpuAddress = gfxAllocation.getUnderlyingBuffer(); - auto gpuAddress = gfxAllocation.getGpuAddress(); - auto size = gfxAllocation.getUnderlyingBufferSize(); - - if (size == 0) + uint64_t gpuAddress; + void *cpuAddress; + size_t size; + if (!this->getParametersForWriteMemory(gfxAllocation, gpuAddress, cpuAddress, size)) { return false; + } - writeMemory(gpuAddress, cpuAddress, size, this->getMemoryBank(&gfxAllocation), this->getPPGTTAdditionalBits(&gfxAllocation), gfxAllocation.devicesBitfield); + if (aubManager) { + this->writeMemoryWithAubManager(gfxAllocation); + } else { + writeMemory(gpuAddress, cpuAddress, size, this->getMemoryBank(&gfxAllocation), this->getPPGTTAdditionalBits(&gfxAllocation), gfxAllocation.devicesBitfield); + } return true; } diff --git a/unit_tests/command_stream/aub_file_stream_tests.cpp b/unit_tests/command_stream/aub_file_stream_tests.cpp index 35abcfc42b..c730a3c698 100644 --- a/unit_tests/command_stream/aub_file_stream_tests.cpp +++ b/unit_tests/command_stream/aub_file_stream_tests.cpp @@ -304,7 +304,6 @@ HWTEST_F(AubFileStreamTests, givenAubCommandStreamReceiverWhenFlushIsCalledThenI auto aubStream = std::make_unique(); MockAubCenter *mockAubCenter = new MockAubCenter(platformDevices[0], false, ""); mockAubCenter->aubManager = std::make_unique(); - auto mockManager = static_cast(mockAubCenter->aubManager.get()); pDevice->executionEnvironment->aubCenter.reset(mockAubCenter); MockAubCsr aubCsr(**platformDevices, "", true, *pDevice->executionEnvironment); @@ -326,9 +325,7 @@ HWTEST_F(AubFileStreamTests, givenAubCommandStreamReceiverWhenFlushIsCalledThenI EXPECT_TRUE(mockHardwareContext->submitCalled); EXPECT_FALSE(mockHardwareContext->pollForCompletionCalled); - //call writeMemory on aubManager to clone page tables - EXPECT_FALSE(mockHardwareContext->writeMemoryCalled); - EXPECT_TRUE(mockManager->writeMemoryCalled); + EXPECT_TRUE(aubCsr.writeMemoryWithAubManagerCalled); pDevice->executionEnvironment->memoryManager->freeGraphicsMemory(commandBuffer); } @@ -360,21 +357,17 @@ HWTEST_F(AubFileStreamTests, givenAubCommandStreamReceiverWhenFlushIsCalledWithZ HWTEST_F(AubFileStreamTests, givenAubCommandStreamReceiverWhenMakeResidentIsCalledThenItShouldCallTheExpectedHwContextFunctions) { MockAubCenter *mockAubCenter = new MockAubCenter(platformDevices[0], false, ""); mockAubCenter->aubManager = std::make_unique(); - auto mockManager = static_cast(mockAubCenter->aubManager.get()); pDevice->executionEnvironment->aubCenter.reset(mockAubCenter); MockAubCsr aubCsr(**platformDevices, "", true, *pDevice->executionEnvironment); OsContext osContext(nullptr, 0, {EngineType::ENGINE_RCS, 0}, PreemptionMode::Disabled); aubCsr.setupContext(osContext); - auto mockHardwareContext = static_cast(aubCsr.hardwareContextController->hardwareContexts[0].get()); MockGraphicsAllocation allocation(reinterpret_cast(0x1000), 0x1000); ResidencyContainer allocationsForResidency = {&allocation}; aubCsr.processResidency(allocationsForResidency); - //call writeMemory on aubManager to clone page tables - EXPECT_FALSE(mockHardwareContext->writeMemoryCalled); - EXPECT_TRUE(mockManager->writeMemoryCalled); + EXPECT_TRUE(aubCsr.writeMemoryWithAubManagerCalled); } HWTEST_F(AubFileStreamTests, givenAubCommandStreamReceiverWhenExpectMemoryEqualIsCalledThenItShouldCallTheExpectedHwContextFunctions) { diff --git a/unit_tests/command_stream/tbx_command_stream_tests.cpp b/unit_tests/command_stream/tbx_command_stream_tests.cpp index 50505fa9a6..34f52692a1 100644 --- a/unit_tests/command_stream/tbx_command_stream_tests.cpp +++ b/unit_tests/command_stream/tbx_command_stream_tests.cpp @@ -343,7 +343,6 @@ HWTEST_F(TbxCommandStreamTests, givenTbxCommandStreamReceiverWhenItIsCreatedWith HWTEST_F(TbxCommandStreamTests, givenTbxCommandStreamReceiverWhenFlushIsCalledThenItShouldCallTheExpectedHwContextFunctions) { MockAubCenter *mockAubCenter = new MockAubCenter(platformDevices[0], false, ""); mockAubCenter->aubManager = std::make_unique(); - auto mockManager = static_cast(mockAubCenter->aubManager.get()); pDevice->executionEnvironment->aubCenter.reset(mockAubCenter); MockTbxCsr tbxCsr(**platformDevices, *pDevice->executionEnvironment); @@ -364,9 +363,7 @@ HWTEST_F(TbxCommandStreamTests, givenTbxCommandStreamReceiverWhenFlushIsCalledTh EXPECT_TRUE(mockHardwareContext->submitCalled); EXPECT_TRUE(mockHardwareContext->pollForCompletionCalled); - //call writeMemory on aubManager to clone page tables - EXPECT_FALSE(mockHardwareContext->writeMemoryCalled); - EXPECT_TRUE(mockManager->writeMemoryCalled); + EXPECT_TRUE(tbxCsr.writeMemoryWithAubManagerCalled); pDevice->executionEnvironment->memoryManager->freeGraphicsMemory(commandBuffer); } @@ -376,13 +373,11 @@ HWTEST_F(TbxCommandStreamTests, givenTbxCommandStreamReceiverInBatchedModeWhenFl MockAubCenter *mockAubCenter = new MockAubCenter(platformDevices[0], false, ""); mockAubCenter->aubManager = std::make_unique(); - auto mockManager = static_cast(mockAubCenter->aubManager.get()); pDevice->executionEnvironment->aubCenter.reset(mockAubCenter); MockTbxCsr tbxCsr(**platformDevices, *pDevice->executionEnvironment); OsContext osContext(nullptr, 0, {EngineType::ENGINE_RCS, 0}, PreemptionMode::Disabled); tbxCsr.setupContext(osContext); - auto mockHardwareContext = static_cast(tbxCsr.hardwareContextController->hardwareContexts[0].get()); auto commandBuffer = pDevice->executionEnvironment->memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{MemoryConstants::pageSize}); @@ -392,9 +387,7 @@ HWTEST_F(TbxCommandStreamTests, givenTbxCommandStreamReceiverInBatchedModeWhenFl tbxCsr.flush(batchBuffer, allocationsForResidency); - //call writeMemory on aubManager to clone page tables - EXPECT_FALSE(mockHardwareContext->writeMemoryCalled); - EXPECT_TRUE(mockManager->writeMemoryCalled); + EXPECT_TRUE(tbxCsr.writeMemoryWithAubManagerCalled); EXPECT_EQ(1u, batchBuffer.commandBufferAllocation->getResidencyTaskCount(tbxCsr.getOsContext().getContextId())); pDevice->executionEnvironment->memoryManager->freeGraphicsMemory(commandBuffer); } @@ -424,21 +417,17 @@ HWTEST_F(TbxCommandStreamTests, givenTbxCommandStreamReceiverWhenFlushIsCalledWi HWTEST_F(TbxCommandStreamTests, givenTbxCommandStreamReceiverWhenMakeResidentIsCalledThenItShouldCallTheExpectedHwContextFunctions) { MockAubCenter *mockAubCenter = new MockAubCenter(platformDevices[0], false, ""); mockAubCenter->aubManager = std::make_unique(); - auto mockManager = static_cast(mockAubCenter->aubManager.get()); pDevice->executionEnvironment->aubCenter.reset(mockAubCenter); MockTbxCsr tbxCsr(**platformDevices, *pDevice->executionEnvironment); OsContext osContext(nullptr, 0, {EngineType::ENGINE_RCS, 0}, PreemptionMode::Disabled); tbxCsr.setupContext(osContext); - auto mockHardwareContext = static_cast(tbxCsr.hardwareContextController->hardwareContexts[0].get()); MockGraphicsAllocation allocation(reinterpret_cast(0x1000), 0x1000); ResidencyContainer allocationsForResidency = {&allocation}; tbxCsr.processResidency(allocationsForResidency); - //call writeMemory on aubManager to clone page tables - EXPECT_FALSE(mockHardwareContext->writeMemoryCalled); - EXPECT_TRUE(mockManager->writeMemoryCalled); + EXPECT_TRUE(tbxCsr.writeMemoryWithAubManagerCalled); } HWTEST_F(TbxCommandStreamTests, givenTbxCommandStreamReceiverWhenMakeCoherentIsCalledThenItShouldCallTheExpectedHwContextFunctions) { diff --git a/unit_tests/mocks/mock_aub_csr.h b/unit_tests/mocks/mock_aub_csr.h index f8945f4da4..8176f65af0 100644 --- a/unit_tests/mocks/mock_aub_csr.h +++ b/unit_tests/mocks/mock_aub_csr.h @@ -57,6 +57,7 @@ struct MockAubCsr : public AUBCommandStreamReceiverHw { using CommandStreamReceiverHw::defaultSshSize; using AUBCommandStreamReceiverHw::taskCount; using AUBCommandStreamReceiverHw::pollForCompletionTaskCount; + using AUBCommandStreamReceiverHw::writeMemory; DispatchMode peekDispatchMode() const { return this->dispatchMode; @@ -80,7 +81,7 @@ struct MockAubCsr : public AUBCommandStreamReceiverHw { AUBCommandStreamReceiverHw::initializeEngine(); initializeEngineCalled = true; } - void writeMemory(uint64_t gpuAddress, void *cpuAddress, size_t size, uint32_t memoryBank, uint64_t entryBits, DevicesBitfield devicesBitfield) { + void writeMemory(uint64_t gpuAddress, void *cpuAddress, size_t size, uint32_t memoryBank, uint64_t entryBits, DevicesBitfield devicesBitfield) override { AUBCommandStreamReceiverHw::writeMemory(gpuAddress, cpuAddress, size, memoryBank, entryBits, devicesBitfield); writeMemoryCalled = true; } @@ -88,6 +89,12 @@ struct MockAubCsr : public AUBCommandStreamReceiverHw { AUBCommandStreamReceiverHw::submitBatchBuffer(batchBufferGpuAddress, batchBuffer, batchBufferSize, memoryBank, entryBits); submitBatchBufferCalled = true; } + + void writeMemoryWithAubManager(GraphicsAllocation &graphicsAllocation) override { + CommandStreamReceiverSimulatedHw::writeMemoryWithAubManager(graphicsAllocation); + writeMemoryWithAubManagerCalled = true; + } + void pollForCompletion() override { AUBCommandStreamReceiverHw::pollForCompletion(); pollForCompletionCalled = true; @@ -107,6 +114,7 @@ struct MockAubCsr : public AUBCommandStreamReceiverHw { bool initProgrammingFlagsCalled = false; bool initializeEngineCalled = false; bool writeMemoryCalled = false; + bool writeMemoryWithAubManagerCalled = false; bool submitBatchBufferCalled = false; bool pollForCompletionCalled = false; bool expectMemoryEqualCalled = false; diff --git a/unit_tests/mocks/mock_tbx_csr.h b/unit_tests/mocks/mock_tbx_csr.h index 2440d3fc10..00e893bd07 100644 --- a/unit_tests/mocks/mock_tbx_csr.h +++ b/unit_tests/mocks/mock_tbx_csr.h @@ -35,6 +35,7 @@ class MockTbxCsrToTestWaitBeforeMakingNonResident : public TbxCommandStreamRecei template class MockTbxCsr : public TbxCommandStreamReceiverHw { public: + using TbxCommandStreamReceiverHw::writeMemory; MockTbxCsr(const HardwareInfo &hwInfoIn, ExecutionEnvironment &executionEnvironment) : TbxCommandStreamReceiverHw(hwInfoIn, executionEnvironment) {} @@ -42,7 +43,13 @@ class MockTbxCsr : public TbxCommandStreamReceiverHw { TbxCommandStreamReceiverHw::initializeEngine(); initializeEngineCalled = true; } - void writeMemory(uint64_t gpuAddress, void *cpuAddress, size_t size, uint32_t memoryBank, uint64_t entryBits, DevicesBitfield devicesBitfield) { + + void writeMemoryWithAubManager(GraphicsAllocation &graphicsAllocation) override { + CommandStreamReceiverSimulatedHw::writeMemoryWithAubManager(graphicsAllocation); + writeMemoryWithAubManagerCalled = true; + } + + void writeMemory(uint64_t gpuAddress, void *cpuAddress, size_t size, uint32_t memoryBank, uint64_t entryBits, DevicesBitfield devicesBitfield) override { TbxCommandStreamReceiverHw::writeMemory(gpuAddress, cpuAddress, size, memoryBank, entryBits, devicesBitfield); writeMemoryCalled = true; } @@ -59,6 +66,7 @@ class MockTbxCsr : public TbxCommandStreamReceiverHw { makeCoherentCalled = true; } bool initializeEngineCalled = false; + bool writeMemoryWithAubManagerCalled = false; bool writeMemoryCalled = false; bool submitBatchBufferCalled = false; bool pollForCompletionCalled = false;