intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-20 00:24:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Return error when failing on submission

Browse Source 
Signed-off-by: Raiyan Latif <raiyan.latif@intel.com>
This commit is contained in:
Raiyan Latif
2022-01-07 14:53:31 +00:00
committed by Compute-Runtime-Automation
parent 38f9df26dd
commit 394c0e90e1
45 changed files with 668 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
level_zero/core/source/cmdqueue/cmdqueue.cpp
View File

@@ -72,8 +72,8 @@ void CommandQueueImp::reserveLinearStreamSize(size_t size) {
}
}
int CommandQueueImp::submitBatchBuffer(size_t offset, NEO::ResidencyContainer &residencyContainer, void *endingCmdPtr,
bool isCooperative) {
NEO::SubmissionStatus CommandQueueImp::submitBatchBuffer(size_t offset, NEO::ResidencyContainer &residencyContainer, void *endingCmdPtr,
bool isCooperative) {
UNRECOVERABLE_IF(csr == nullptr);
NEO::BatchBuffer batchBuffer(commandStream->getGraphicsAllocation(), offset, 0u, nullptr, false, false,
@@ -85,7 +85,7 @@ int CommandQueueImp::submitBatchBuffer(size_t offset, NEO::ResidencyContainer &r
csr->setActivePartitions(partitionCount);
auto ret = csr->submitBatchBuffer(batchBuffer, csr->getResidencyAllocations());
if (ret) {
if (ret != NEO::SubmissionStatus::SUCCESS) {
return ret;
}

9
level_zero/core/source/cmdqueue/cmdqueue_hw.inl
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -14,6 +14,7 @@
#include "shared/source/command_stream/command_stream_receiver_hw.h"
#include "shared/source/command_stream/linear_stream.h"
#include "shared/source/command_stream/preemption.h"
#include "shared/source/command_stream/submission_status.h"
#include "shared/source/command_stream/thread_arbitration_policy.h"
#include "shared/source/device/device.h"
#include "shared/source/helpers/hw_helper.h"
@@ -480,7 +481,11 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandLists(
this->heapContainer.clear();
csr->pollForCompletion();
if (ret) {
if (ret != NEO::SubmissionStatus::SUCCESS) {
if (ret == NEO::SubmissionStatus::OUT_OF_MEMORY) {
return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY;
}
return ZE_RESULT_ERROR_UNKNOWN;
}

7
level_zero/core/source/cmdqueue/cmdqueue_imp.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -8,6 +8,7 @@
#pragma once
#include "shared/source/command_stream/csr_definitions.h"
#include "shared/source/command_stream/submission_status.h"
#include "shared/source/command_stream/submissions_aggregator.h"
#include "shared/source/helpers/constants.h"
#include "shared/source/indirect_heap/indirect_heap.h"
@@ -83,8 +84,8 @@ struct CommandQueueImp : public CommandQueue {
virtual bool getPreemptionCmdProgramming() = 0;
protected:
MOCKABLE_VIRTUAL int submitBatchBuffer(size_t offset, NEO::ResidencyContainer &residencyContainer, void *endingCmdPtr,
bool isCooperative);
MOCKABLE_VIRTUAL NEO::SubmissionStatus submitBatchBuffer(size_t offset, NEO::ResidencyContainer &residencyContainer, void *endingCmdPtr,
bool isCooperative);
ze_result_t synchronizeByPollingForTaskCount(uint64_t timeout);

4
level_zero/core/test/unit_tests/mocks/mock_cmdqueue.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -72,7 +72,7 @@ struct MockCommandQueueHw : public L0::CommandQueueHw<gfxCoreFamily> {
return ZE_RESULT_SUCCESS;
}
int submitBatchBuffer(size_t offset, NEO::ResidencyContainer &residencyContainer, void *endingCmdPtr, bool isCooperative) override {
NEO::SubmissionStatus submitBatchBuffer(size_t offset, NEO::ResidencyContainer &residencyContainer, void *endingCmdPtr, bool isCooperative) override {
residencyContainerSnapshot = residencyContainer;
return BaseClass::submitBatchBuffer(offset, residencyContainer, endingCmdPtr, isCooperative);
}

92
level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue_1.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -291,8 +291,27 @@ HWTEST_F(CommandQueueCreate, givenCommandStreamReceiverFailsThenSubmitBatchBuffe
false,
returnValue));
ResidencyContainer container;
int ret = commandQueue->submitBatchBuffer(0, container, nullptr, false);
EXPECT_NE(ret, 0);
NEO::SubmissionStatus ret = commandQueue->submitBatchBuffer(0, container, nullptr, false);
EXPECT_EQ(ret, NEO::SubmissionStatus::FAILED);
commandQueue->destroy();
}
HWTEST_F(CommandQueueCreate, givenOutOfMemoryThenSubmitBatchBufferReturnsOutOfMemoryError) {
auto csr = std::make_unique<MockCommandStreamReceiverWithOutOfMemorySubmitBatch>(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield());
csr->setupContext(*neoDevice->getDefaultEngine().osContext);
const ze_command_queue_desc_t desc = {};
ze_result_t returnValue;
auto commandQueue = whitebox_cast(CommandQueue::create(productFamily,
device,
csr.get(),
&desc,
false,
false,
returnValue));
ResidencyContainer container;
NEO::SubmissionStatus ret = commandQueue->submitBatchBuffer(0, container, nullptr, false);
EXPECT_EQ(ret, NEO::SubmissionStatus::OUT_OF_MEMORY);
commandQueue->destroy();
}
@@ -1064,6 +1083,73 @@ HWTEST2_F(ExecuteCommandListTests, givenExecuteCommandListWhenItReturnsThenConta
alignedFree(alloc);
}
template <GFXCORE_FAMILY gfxCoreFamily>
class MockCommandQueueSubmitBatchBuffer : public MockCommandQueue<gfxCoreFamily> {
public:
MockCommandQueueSubmitBatchBuffer(L0::Device *device, NEO::CommandStreamReceiver *csr, const ze_command_queue_desc_t *desc) : MockCommandQueue<gfxCoreFamily>(device, csr, desc) {}
ADDMETHOD_NOBASE(submitBatchBuffer, NEO::SubmissionStatus, NEO::SubmissionStatus::SUCCESS,
(size_t offset, NEO::ResidencyContainer &residencyContainer, void *endingCmdPtr,
bool isCooperative));
};
HWTEST2_F(ExecuteCommandListTests, givenOutOfMemorySubmitBatchBufferThenExecuteCommandListReturnsOutOfMemoryError, IsAtLeastSkl) {
ze_command_queue_desc_t desc = {};
NEO::CommandStreamReceiver *csr;
device->getCsrForOrdinalAndIndex(&csr, 0u, 0u);
auto commandQueue = new MockCommandQueueSubmitBatchBuffer<gfxCoreFamily>(device, csr, &desc);
commandQueue->submitBatchBufferResult = NEO::SubmissionStatus::OUT_OF_MEMORY;
commandQueue->initialize(false, false);
auto commandList = new CommandListCoreFamily<gfxCoreFamily>();
commandList->initialize(device, NEO::EngineGroupType::Compute, 0u);
auto commandListHandle = commandList->toHandle();
auto res = commandQueue->executeCommandLists(1, &commandListHandle, nullptr, false);
EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, res);
commandQueue->destroy();
commandList->destroy();
}
HWTEST2_F(ExecuteCommandListTests, givenFailingSubmitBatchBufferThenExecuteCommandListReturnsErrorUnknown, IsAtLeastSkl) {
ze_command_queue_desc_t desc = {};
NEO::CommandStreamReceiver *csr;
device->getCsrForOrdinalAndIndex(&csr, 0u, 0u);
auto commandQueue = new MockCommandQueueSubmitBatchBuffer<gfxCoreFamily>(device, csr, &desc);
commandQueue->submitBatchBufferResult = NEO::SubmissionStatus::FAILED;
commandQueue->initialize(false, false);
auto commandList = new CommandListCoreFamily<gfxCoreFamily>();
commandList->initialize(device, NEO::EngineGroupType::Compute, 0u);
auto commandListHandle = commandList->toHandle();
auto res = commandQueue->executeCommandLists(1, &commandListHandle, nullptr, false);
EXPECT_EQ(ZE_RESULT_ERROR_UNKNOWN, res);
commandQueue->destroy();
commandList->destroy();
}
HWTEST2_F(ExecuteCommandListTests, givenSuccessfulSubmitBatchBufferThenExecuteCommandListReturnsSuccess, IsAtLeastSkl) {
ze_command_queue_desc_t desc = {};
NEO::CommandStreamReceiver *csr;
device->getCsrForOrdinalAndIndex(&csr, 0u, 0u);
auto commandQueue = new MockCommandQueueSubmitBatchBuffer<gfxCoreFamily>(device, csr, &desc);
commandQueue->submitBatchBufferResult = NEO::SubmissionStatus::SUCCESS;
commandQueue->initialize(false, false);
auto commandList = new CommandListCoreFamily<gfxCoreFamily>();
commandList->initialize(device, NEO::EngineGroupType::Compute, 0u);
auto commandListHandle = commandList->toHandle();
auto res = commandQueue->executeCommandLists(1, &commandListHandle, nullptr, false);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
commandQueue->destroy();
commandList->destroy();
}
HWTEST2_F(ExecuteCommandListTests, givenCommandQueueHavingTwoB2BCommandListsThenMVSDirtyFlagAndGSBADirtyFlagAreSetOnlyOnce, IsAtLeastSkl) {
ze_command_queue_desc_t desc = {};
NEO::CommandStreamReceiver *csr;

4
level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue_enqueue_cmdlist.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -510,7 +510,7 @@ HWTEST2_F(CommandQueueExecuteCommandLists, givenCommandListsWithCooperativeAndNo
HWTEST2_F(CommandQueueExecuteCommandLists, givenCommandListWithCooperativeKernelsWhenExecuteCommandListsIsCalledThenCorrectBatchBufferIsSubmitted, IsAtLeastXeHpCore) {
struct MockCsr : NEO::CommandStreamReceiverHw<FamilyType> {
using NEO::CommandStreamReceiverHw<FamilyType>::CommandStreamReceiverHw;
int submitBatchBuffer(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
NEO::SubmissionStatus submitBatchBuffer(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
useSingleSubdeviceValue = batchBuffer.useSingleSubdevice;
submitBatchBufferCalled++;
return NEO::CommandStreamReceiver::submitBatchBuffer(batchBuffer, allocationsForResidency);

4
opencl/test/unit_test/command_queue/blit_enqueue_tests.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2021 Intel Corporation
* Copyright (C) 2019-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -1285,7 +1285,7 @@ struct BlitEnqueueFlushTests : public BlitEnqueueTests<1> {
public:
using UltCommandStreamReceiver<FamilyType>::UltCommandStreamReceiver;
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
latestFlushedCounter = ++(*flushCounter);
return UltCommandStreamReceiver<FamilyType>::flush(batchBuffer, allocationsForResidency);
}

6
opencl/test/unit_test/command_queue/enqueue_thread_tests.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -38,7 +38,7 @@ class CommandStreamReceiverMock : public UltCommandStreamReceiver<FamilyType> {
this->pClDevice = pDevice->getSpecializedDevice<ClDevice>();
}
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
EXPECT_NE(nullptr, batchBuffer.commandBufferAllocation->getUnderlyingBuffer());
toFree.push_back(batchBuffer.commandBufferAllocation);
@@ -47,7 +47,7 @@ class CommandStreamReceiverMock : public UltCommandStreamReceiver<FamilyType> {
EXPECT_TRUE(this->ownershipMutex.try_lock());
this->ownershipMutex.unlock();
return true;
return SubmissionStatus::SUCCESS;
}
~CommandStreamReceiverMock() override {

10
opencl/test/unit_test/command_stream/command_stream_receiver_flush_task_1_tests.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -247,11 +247,11 @@ HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCsrInDefaultModeAndMidThreadP
HWTEST_F(CommandStreamReceiverFlushTaskTests, whenFlushThenCommandBufferAlreadyHasProperTaskCountsAndIsNotIncludedInResidencyVector) {
struct MockCsrFlushCmdBuffer : public MockCommandStreamReceiver {
using MockCommandStreamReceiver::MockCommandStreamReceiver;
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
NEO::SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
EXPECT_EQ(batchBuffer.commandBufferAllocation->getResidencyTaskCount(this->osContext->getContextId()), this->taskCount + 1);
EXPECT_EQ(batchBuffer.commandBufferAllocation->getTaskCount(this->osContext->getContextId()), this->taskCount + 1);
EXPECT_EQ(std::find(allocationsForResidency.begin(), allocationsForResidency.end(), batchBuffer.commandBufferAllocation), allocationsForResidency.end());
return true;
return NEO::SubmissionStatus::SUCCESS;
}
};
@@ -1050,9 +1050,9 @@ struct CommandStreamReceiverHwLog : public UltCommandStreamReceiver<FamilyType>
flushCount(0) {
}
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
++flushCount;
return true;
return SubmissionStatus::SUCCESS;
}
int flushCount;

6
opencl/test/unit_test/command_stream/command_stream_receiver_flush_task_3_tests.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -1873,8 +1873,8 @@ class MockCsrWithFailingFlush : public CommandStreamReceiverHw<GfxFamily> {
this->dispatchMode = DispatchMode::BatchedDispatch;
this->tagAddress = &tag;
}
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
return false;
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
return SubmissionStatus::FAILED;
}
uint32_t tag = 0;
};

6
opencl/test/unit_test/command_stream/command_stream_receiver_with_aub_dump_tests.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -42,14 +42,14 @@ struct MyMockCsr : UltCommandStreamReceiver<DEFAULT_TEST_FAMILY_NAME> {
: UltCommandStreamReceiver(executionEnvironment, rootDeviceIndex, deviceBitfield) {
}
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
flushParametrization.wasCalled = true;
flushParametrization.receivedBatchBuffer = &batchBuffer;
flushParametrization.receivedEngine = osContext->getEngineType();
flushParametrization.receivedAllocationsForResidency = &allocationsForResidency;
processResidency(allocationsForResidency, 0u);
flushStamp->setStamp(flushParametrization.flushStampToReturn);
return true;
return SubmissionStatus::SUCCESS;
}
void makeResident(GraphicsAllocation &gfxAllocation) override {

4
opencl/test/unit_test/kernel/kernel_tests.cpp
View File

@@ -483,8 +483,8 @@ class CommandStreamReceiverMock : public CommandStreamReceiver {
}
}
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
return true;
NEO::SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
return NEO::SubmissionStatus::SUCCESS;
}
void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool quickKmdSleep, bool forcePowerSavingMode) override {

54
opencl/test/unit_test/os_interface/linux/drm_command_stream_tests_1.cpp
View File

@@ -824,6 +824,60 @@ HWTEST_TEMPLATED_F(DrmCommandStreamDirectSubmissionTest, givenEnabledDirectSubmi
EXPECT_FALSE(directSubmission->ringStart);
}
template <typename GfxFamily>
struct MockDrmDirectSubmissionDispatchCommandBuffer : public DrmDirectSubmission<GfxFamily, RenderDispatcher<GfxFamily>> {
MockDrmDirectSubmissionDispatchCommandBuffer<GfxFamily>(Device &device, OsContext &osContext)
: DrmDirectSubmission<GfxFamily, RenderDispatcher<GfxFamily>>(device, osContext) {
}
ADDMETHOD_NOBASE(dispatchCommandBuffer, bool, false,
(BatchBuffer & batchBuffer, FlushStampTracker &flushStamp));
};
template <typename GfxFamily>
struct MockDrmBlitterDirectSubmissionDispatchCommandBuffer : public DrmDirectSubmission<GfxFamily, BlitterDispatcher<GfxFamily>> {
MockDrmBlitterDirectSubmissionDispatchCommandBuffer<GfxFamily>(Device &device, OsContext &osContext)
: DrmDirectSubmission<GfxFamily, BlitterDispatcher<GfxFamily>>(device, osContext) {
}
ADDMETHOD_NOBASE(dispatchCommandBuffer, bool, false,
(BatchBuffer & batchBuffer, FlushStampTracker &flushStamp));
};
HWTEST_TEMPLATED_F(DrmCommandStreamDirectSubmissionTest, givenDirectSubmissionFailsThenFlushReturnsError) {
static_cast<TestedDrmCommandStreamReceiver<FamilyType> *>(csr)->directSubmission = std::make_unique<MockDrmDirectSubmissionDispatchCommandBuffer<FamilyType>>(*device.get(), *device->getDefaultEngine().osContext);
auto directSubmission = static_cast<TestedDrmCommandStreamReceiver<FamilyType> *>(csr)->directSubmission.get();
static_cast<MockDrmDirectSubmissionDispatchCommandBuffer<FamilyType> *>(directSubmission)->dispatchCommandBufferResult = false;
auto &cs = csr->getCS();
CommandStreamReceiverHw<FamilyType>::addBatchBufferEnd(cs, nullptr);
EncodeNoop<FamilyType>::alignToCacheLine(cs);
BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 4, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr, false};
uint8_t bbStart[64];
batchBuffer.endCmdPtr = &bbStart[0];
auto res = csr->flush(batchBuffer, csr->getResidencyAllocations());
EXPECT_GT(static_cast<MockDrmDirectSubmissionDispatchCommandBuffer<FamilyType> *>(directSubmission)->dispatchCommandBufferCalled, 0u);
EXPECT_EQ(NEO::SubmissionStatus::FAILED, res);
}
HWTEST_TEMPLATED_F(DrmCommandStreamBlitterDirectSubmissionTest, givenBlitterDirectSubmissionFailsThenFlushReturnsError) {
static_cast<TestedDrmCommandStreamReceiver<FamilyType> *>(csr)->blitterDirectSubmission = std::make_unique<MockDrmBlitterDirectSubmissionDispatchCommandBuffer<FamilyType>>(*device.get(), *device->getDefaultEngine().osContext);
auto blitterDirectSubmission = static_cast<TestedDrmCommandStreamReceiver<FamilyType> *>(csr)->blitterDirectSubmission.get();
static_cast<MockDrmBlitterDirectSubmissionDispatchCommandBuffer<FamilyType> *>(blitterDirectSubmission)->dispatchCommandBufferResult = false;
auto &cs = csr->getCS();
CommandStreamReceiverHw<FamilyType>::addBatchBufferEnd(cs, nullptr);
EncodeNoop<FamilyType>::alignToCacheLine(cs);
BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 4, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr, false};
uint8_t bbStart[64];
batchBuffer.endCmdPtr = &bbStart[0];
auto res = csr->flush(batchBuffer, csr->getResidencyAllocations());
EXPECT_GT(static_cast<MockDrmBlitterDirectSubmissionDispatchCommandBuffer<FamilyType> *>(blitterDirectSubmission)->dispatchCommandBufferCalled, 0u);
EXPECT_EQ(NEO::SubmissionStatus::FAILED, res);
}
template <typename GfxFamily>
struct MockDrmDirectSubmission : public DrmDirectSubmission<GfxFamily, RenderDispatcher<GfxFamily>> {
using DrmDirectSubmission<GfxFamily, RenderDispatcher<GfxFamily>>::currentTagData;

259
opencl/test/unit_test/os_interface/linux/drm_command_stream_tests_2.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -637,8 +637,8 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedWithFailingExec, GivenFailingExecThen
CommandStreamReceiverHw<FamilyType>::addBatchBufferEnd(cs, nullptr);
EncodeNoop<FamilyType>::alignToCacheLine(cs);
BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr, false};
bool ret = csr->flush(batchBuffer, csr->getResidencyAllocations());
EXPECT_EQ(ret, false);
NEO::SubmissionStatus ret = csr->flush(batchBuffer, csr->getResidencyAllocations());
EXPECT_EQ(ret, NEO::SubmissionStatus::FAILED);
}
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, GivenNotAlignedWhenFlushingThenSucceeds) {
@@ -790,6 +790,123 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenDrmCommandStreamReceiverWh
EXPECT_TRUE(mm->isValidateHostMemoryEnabled());
}
struct MockDrmAllocationBindBO : public DrmAllocation {
MockDrmAllocationBindBO(uint32_t rootDeviceIndex, AllocationType allocationType, BufferObjects &bos, void *ptrIn, uint64_t gpuAddress, size_t sizeIn, MemoryPool::Type pool)
: DrmAllocation(rootDeviceIndex, allocationType, bos, ptrIn, gpuAddress, sizeIn, pool) {
}
ADDMETHOD_NOBASE(bindBO, int, 0,
(BufferObject * bo, OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind));
};
struct MockDrmAllocationBindBOs : public DrmAllocation {
MockDrmAllocationBindBOs(uint32_t rootDeviceIndex, AllocationType allocationType, BufferObjects &bos, void *ptrIn, uint64_t gpuAddress, size_t sizeIn, MemoryPool::Type pool)
: DrmAllocation(rootDeviceIndex, allocationType, bos, ptrIn, gpuAddress, sizeIn, pool) {
}
ADDMETHOD_NOBASE(bindBOs, int, 0,
(OsContext * osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind));
};
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenBindBOsFailsThenMakeBOsResidentReturnsError) {
auto size = 1024u;
auto bo = this->createBO(size);
BufferObjects bos{bo};
auto allocation = new MockDrmAllocationBindBOs(0, GraphicsAllocation::AllocationType::UNKNOWN, bos, nullptr, 0u, size, MemoryPool::LocalMemory);
allocation->bindBOsResult = -1;
auto res = allocation->makeBOsResident(&csr->getOsContext(), 0, nullptr, true);
EXPECT_NE(res, 0);
EXPECT_EQ(allocation->fragmentsStorage.fragmentCount, 0u);
EXPECT_GT(allocation->bindBOsCalled, 0u);
mm->freeGraphicsMemory(allocation);
}
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenFragmentStorageAndBindBOFailsThenMakeBOsResidentReturnsError) {
auto size = 1024u;
auto bo = this->createBO(size);
BufferObjects bos{bo};
auto allocation = new MockDrmAllocationBindBO(0, GraphicsAllocation::AllocationType::UNKNOWN, bos, nullptr, 0u, size, MemoryPool::LocalMemory);
allocation->bindBOResult = -1;
OsHandleStorage prevStorage;
OsHandleStorage storage;
OsHandleLinux osHandleStorage;
ResidencyData *residency = new ResidencyData(1);
storage.fragmentCount = 1;
storage.fragmentStorageData[0].osHandleStorage = &osHandleStorage;
storage.fragmentStorageData[0].residency = residency;
storage.fragmentStorageData[0].residency->resident[csr->getOsContext().getContextId()] = false;
memcpy(&prevStorage, &allocation->fragmentsStorage, sizeof(OsHandleStorage));
memcpy(&allocation->fragmentsStorage, &storage, sizeof(OsHandleStorage));
auto res = allocation->makeBOsResident(&csr->getOsContext(), 0, nullptr, true);
EXPECT_NE(res, 0);
EXPECT_EQ(allocation->fragmentsStorage.fragmentCount, 1u);
EXPECT_GT(allocation->bindBOCalled, 0u);
memcpy(&allocation->fragmentsStorage, &prevStorage, sizeof(OsHandleStorage));
mm->freeGraphicsMemory(allocation);
delete residency;
}
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenBindBOFailsThenBindBOsReturnsError) {
auto size = 1024u;
auto bo = this->createBO(size);
BufferObjects bos{bo};
auto allocation = new MockDrmAllocationBindBO(0, GraphicsAllocation::AllocationType::UNKNOWN, bos, nullptr, 0u, size, MemoryPool::LocalMemory);
allocation->bindBOResult = -1;
auto res = allocation->bindBOs(&csr->getOsContext(), 0u, &static_cast<TestedDrmCommandStreamReceiver<FamilyType> *>(csr)->residency, false);
EXPECT_NE(res, 0);
EXPECT_GT(allocation->bindBOCalled, 0u);
mm->freeGraphicsMemory(allocation);
}
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenBindBOFailsWithMultipleMemoryBanksThenBindBOsReturnsError) {
auto size = 1024u;
auto bo = this->createBO(size);
auto bo2 = this->createBO(size);
BufferObjects bos{bo, bo2};
auto allocation = new MockDrmAllocationBindBO(0, GraphicsAllocation::AllocationType::UNKNOWN, bos, nullptr, 0u, size, MemoryPool::LocalMemory);
allocation->bindBOResult = -1;
allocation->storageInfo.memoryBanks = 0b11;
EXPECT_EQ(allocation->storageInfo.getNumBanks(), 2u);
auto res = allocation->bindBOs(&csr->getOsContext(), 0u, &static_cast<TestedDrmCommandStreamReceiver<FamilyType> *>(csr)->residency, false);
EXPECT_NE(res, 0);
EXPECT_GT(allocation->bindBOCalled, 0u);
mm->freeGraphicsMemory(allocation);
}
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenBindBOFailsWithMultipleMemoryBanksWithTileInstancedThenBindBOsReturnsError) {
auto size = 1024u;
auto bo = this->createBO(size);
auto bo2 = this->createBO(size);
BufferObjects bos{bo, bo2};
auto allocation = new MockDrmAllocationBindBO(0, GraphicsAllocation::AllocationType::UNKNOWN, bos, nullptr, 0u, size, MemoryPool::LocalMemory);
allocation->bindBOResult = -1;
allocation->storageInfo.tileInstanced = true;
allocation->storageInfo.memoryBanks = 0b11;
EXPECT_EQ(allocation->storageInfo.getNumBanks(), 2u);
auto res = allocation->bindBOs(&csr->getOsContext(), 0u, &static_cast<TestedDrmCommandStreamReceiver<FamilyType> *>(csr)->residency, false);
EXPECT_NE(res, 0);
EXPECT_GT(allocation->bindBOCalled, 0u);
mm->freeGraphicsMemory(allocation);
}
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenAllocationWithSingleBufferObjectWhenMakeResidentBufferObjectsIsCalledThenTheBufferObjectIsMadeResident) {
auto size = 1024u;
auto bo = this->createBO(size);
@@ -1199,6 +1316,142 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenVmBindAvailableUseWaitCall
EXPECT_EQ(1u, mock->isVmBindAvailableCall.called);
}
struct MockMergeResidencyContainerMemoryOperationsHandler : public DrmMemoryOperationsHandlerDefault {
using DrmMemoryOperationsHandlerDefault::DrmMemoryOperationsHandlerDefault;
ADDMETHOD_NOBASE(mergeWithResidencyContainer, NEO::MemoryOperationsStatus, NEO::MemoryOperationsStatus::SUCCESS,
(OsContext * osContext, ResidencyContainer &residencyContainer));
ADDMETHOD_NOBASE(makeResidentWithinOsContext, NEO::MemoryOperationsStatus, NEO::MemoryOperationsStatus::SUCCESS,
(OsContext * osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable));
};
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenMakeResidentWithinOsContextFailsThenFlushReturnsError) {
struct MockDrmCsr : public DrmCommandStreamReceiver<FamilyType> {
using DrmCommandStreamReceiver<FamilyType>::DrmCommandStreamReceiver;
int flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency) override {
return 0;
}
};
mock->bindAvailable = true;
executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface.reset(new MockMergeResidencyContainerMemoryOperationsHandler());
auto operationHandler = static_cast<MockMergeResidencyContainerMemoryOperationsHandler *>(executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface.get());
operationHandler->makeResidentWithinOsContextResult = NEO::MemoryOperationsStatus::FAILED;
auto osContext = std::make_unique<OsContextLinux>(*mock, 0u,
EngineDescriptorHelper::getDefaultDescriptor(HwHelper::get(defaultHwInfo->platform.eRenderCoreFamily).getGpgpuEngineInstances(*defaultHwInfo)[0],
PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo)));
auto commandBuffer = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize});
LinearStream cs(commandBuffer);
CommandStreamReceiverHw<FamilyType>::addBatchBufferEnd(cs, nullptr);
EncodeNoop<FamilyType>::alignToCacheLine(cs);
BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr, false};
MockDrmCsr mockCsr(*executionEnvironment, rootDeviceIndex, 1, gemCloseWorkerMode::gemCloseWorkerInactive);
mockCsr.setupContext(*osContext.get());
auto res = mockCsr.flush(batchBuffer, mockCsr.getResidencyAllocations());
EXPECT_GT(operationHandler->makeResidentWithinOsContextCalled, 0u);
EXPECT_EQ(NEO::SubmissionStatus::FAILED, res);
mm->freeGraphicsMemory(commandBuffer);
}
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenMakeResidentWithinOsContextOutOfMemoryThenFlushReturnsError) {
struct MockDrmCsr : public DrmCommandStreamReceiver<FamilyType> {
using DrmCommandStreamReceiver<FamilyType>::DrmCommandStreamReceiver;
int flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency) override {
return 0;
}
};
mock->bindAvailable = true;
executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface.reset(new MockMergeResidencyContainerMemoryOperationsHandler());
auto operationHandler = static_cast<MockMergeResidencyContainerMemoryOperationsHandler *>(executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface.get());
operationHandler->makeResidentWithinOsContextResult = NEO::MemoryOperationsStatus::OUT_OF_MEMORY;
auto osContext = std::make_unique<OsContextLinux>(*mock, 0u,
EngineDescriptorHelper::getDefaultDescriptor(HwHelper::get(defaultHwInfo->platform.eRenderCoreFamily).getGpgpuEngineInstances(*defaultHwInfo)[0],
PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo)));
auto commandBuffer = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize});
LinearStream cs(commandBuffer);
CommandStreamReceiverHw<FamilyType>::addBatchBufferEnd(cs, nullptr);
EncodeNoop<FamilyType>::alignToCacheLine(cs);
BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr, false};
MockDrmCsr mockCsr(*executionEnvironment, rootDeviceIndex, 1, gemCloseWorkerMode::gemCloseWorkerInactive);
mockCsr.setupContext(*osContext.get());
auto res = mockCsr.flush(batchBuffer, mockCsr.getResidencyAllocations());
EXPECT_GT(operationHandler->makeResidentWithinOsContextCalled, 0u);
EXPECT_EQ(NEO::SubmissionStatus::OUT_OF_MEMORY, res);
mm->freeGraphicsMemory(commandBuffer);
}
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenMergeWithResidencyContainerFailsThenFlushReturnsError) {
struct MockDrmCsr : public DrmCommandStreamReceiver<FamilyType> {
using DrmCommandStreamReceiver<FamilyType>::DrmCommandStreamReceiver;
int flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency) override {
return 0;
}
};
executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface.reset(new MockMergeResidencyContainerMemoryOperationsHandler());
auto operationHandler = static_cast<MockMergeResidencyContainerMemoryOperationsHandler *>(executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface.get());
operationHandler->mergeWithResidencyContainerResult = NEO::MemoryOperationsStatus::FAILED;
auto osContext = std::make_unique<OsContextLinux>(*mock, 0u,
EngineDescriptorHelper::getDefaultDescriptor(HwHelper::get(defaultHwInfo->platform.eRenderCoreFamily).getGpgpuEngineInstances(*defaultHwInfo)[0],
PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo)));
auto commandBuffer = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize});
LinearStream cs(commandBuffer);
CommandStreamReceiverHw<FamilyType>::addBatchBufferEnd(cs, nullptr);
EncodeNoop<FamilyType>::alignToCacheLine(cs);
BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr, false};
MockDrmCsr mockCsr(*executionEnvironment, rootDeviceIndex, 1, gemCloseWorkerMode::gemCloseWorkerInactive);
mockCsr.setupContext(*osContext.get());
auto res = mockCsr.flush(batchBuffer, mockCsr.getResidencyAllocations());
EXPECT_GT(operationHandler->mergeWithResidencyContainerCalled, 0u);
EXPECT_EQ(NEO::SubmissionStatus::FAILED, res);
mm->freeGraphicsMemory(commandBuffer);
}
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenMergeWithResidencyContainerReturnsOutOfMemoryThenFlushReturnsError) {
struct MockDrmCsr : public DrmCommandStreamReceiver<FamilyType> {
using DrmCommandStreamReceiver<FamilyType>::DrmCommandStreamReceiver;
int flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency) override {
return 0;
}
};
executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface.reset(new MockMergeResidencyContainerMemoryOperationsHandler());
auto operationHandler = static_cast<MockMergeResidencyContainerMemoryOperationsHandler *>(executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface.get());
operationHandler->mergeWithResidencyContainerResult = NEO::MemoryOperationsStatus::OUT_OF_MEMORY;
auto osContext = std::make_unique<OsContextLinux>(*mock, 0u,
EngineDescriptorHelper::getDefaultDescriptor(HwHelper::get(defaultHwInfo->platform.eRenderCoreFamily).getGpgpuEngineInstances(*defaultHwInfo)[0],
PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo)));
auto commandBuffer = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize});
LinearStream cs(commandBuffer);
CommandStreamReceiverHw<FamilyType>::addBatchBufferEnd(cs, nullptr);
EncodeNoop<FamilyType>::alignToCacheLine(cs);
BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr, false};
MockDrmCsr mockCsr(*executionEnvironment, rootDeviceIndex, 1, gemCloseWorkerMode::gemCloseWorkerInactive);
mockCsr.setupContext(*osContext.get());
auto res = mockCsr.flush(batchBuffer, mockCsr.getResidencyAllocations());
EXPECT_GT(operationHandler->mergeWithResidencyContainerCalled, 0u);
EXPECT_EQ(NEO::SubmissionStatus::OUT_OF_MEMORY, res);
mm->freeGraphicsMemory(commandBuffer);
}
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenNoAllocsInMemoryOperationHandlerDefaultWhenFlushThenDrmMemoryOperationHandlerIsNotLocked) {
struct MockDrmMemoryOperationsHandler : public DrmMemoryOperationsHandler {
using DrmMemoryOperationsHandler::mutex;

50
opencl/test/unit_test/os_interface/windows/device_command_stream_tests.cpp
View File

@@ -1045,6 +1045,56 @@ HWTEST_F(WddmCsrCompressionTests, givenDisabledCompressionWhenFlushingThenDontIn
memoryManager->freeGraphicsMemory(graphicsAllocation);
}
template <typename GfxFamily>
struct MockWddmDrmDirectSubmissionDispatchCommandBuffer : public MockWddmDirectSubmission<GfxFamily, RenderDispatcher<GfxFamily>> {
MockWddmDrmDirectSubmissionDispatchCommandBuffer<GfxFamily>(Device &device, OsContext &osContext)
: MockWddmDirectSubmission<GfxFamily, RenderDispatcher<GfxFamily>>(device, osContext) {
}
bool dispatchCommandBuffer(BatchBuffer &batchBuffer, FlushStampTracker &flushStamp) override {
dispatchCommandBufferCalled++;
return false;
}
uint32_t dispatchCommandBufferCalled = 0;
};
TEST_F(WddmCommandStreamMockGdiTest, givenDirectSubmissionFailsThenFlushReturnsError) {
using MockSubmission = MockWddmDrmDirectSubmissionDispatchCommandBuffer<DEFAULT_TEST_FAMILY_NAME>;
DebugManager.flags.EnableDirectSubmission.set(1);
auto hwInfo = device->getRootDeviceEnvironment().getMutableHardwareInfo();
hwInfo->capabilityTable.directSubmissionEngines.data[aub_stream::ENGINE_RCS].engineSupported = true;
std::unique_ptr<OsContext> osContext;
osContext.reset(OsContext::create(device->getExecutionEnvironment()->rootDeviceEnvironments[0]->osInterface.get(), 0,
EngineDescriptorHelper::getDefaultDescriptor({aub_stream::ENGINE_RCS, EngineUsage::Regular}, PreemptionMode::ThreadGroup, device->getDeviceBitfield())));
osContext->setDefaultContext(true);
csr->callParentInitDirectSubmission = false;
bool ret = csr->initDirectSubmission(*device.get(), *osContext.get());
EXPECT_TRUE(ret);
EXPECT_TRUE(csr->isDirectSubmissionEnabled());
EXPECT_FALSE(csr->isBlitterDirectSubmissionEnabled());
GraphicsAllocation *commandBuffer = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize});
ASSERT_NE(nullptr, commandBuffer);
LinearStream cs(commandBuffer);
BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0,
nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(),
&cs, commandBuffer->getUnderlyingBuffer(), false};
csr->directSubmission = std::make_unique<MockSubmission>(*device.get(), *osContext.get());
auto res = csr->flush(batchBuffer, csr->getResidencyAllocations());
EXPECT_EQ(NEO::SubmissionStatus::FAILED, res);
auto directSubmission = reinterpret_cast<MockSubmission *>(csr->directSubmission.get());
EXPECT_GT(directSubmission->dispatchCommandBufferCalled, 0u);
memoryManager->freeGraphicsMemory(commandBuffer);
}
TEST_F(WddmCommandStreamMockGdiTest, givenDirectSubmissionEnabledOnRcsWhenFlushingCommandBufferThenExpectDirectSubmissionUsed) {
using Dispatcher = RenderDispatcher<DEFAULT_TEST_FAMILY_NAME>;
using MockSubmission =

3
shared/source/command_stream/CMakeLists.txt
View File

@@ -1,5 +1,5 @@
#
# Copyright (C) 2019-2021 Intel Corporation
# Copyright (C) 2019-2022 Intel Corporation
#
# SPDX-License-Identifier: MIT
#
@@ -50,6 +50,7 @@ set(NEO_CORE_COMMAND_STREAM
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}stream_properties.cpp
${CMAKE_CURRENT_SOURCE_DIR}/stream_properties.h
${CMAKE_CURRENT_SOURCE_DIR}/stream_property.h
${CMAKE_CURRENT_SOURCE_DIR}/submission_status.h
${CMAKE_CURRENT_SOURCE_DIR}/submissions_aggregator.cpp
${CMAKE_CURRENT_SOURCE_DIR}/submissions_aggregator.h
${CMAKE_CURRENT_SOURCE_DIR}/tbx_command_stream_receiver.cpp

5
shared/source/command_stream/aub_command_stream_receiver_hw.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -9,6 +9,7 @@
#include "shared/source/aub/aub_center.h"
#include "shared/source/command_stream/aub_command_stream_receiver.h"
#include "shared/source/command_stream/command_stream_receiver_simulated_hw.h"
#include "shared/source/command_stream/submission_status.h"
#include "shared/source/helpers/array_count.h"
#include "shared/source/memory_manager/os_agnostic_memory_manager.h"
#include "shared/source/memory_manager/page_table.h"
@@ -37,7 +38,7 @@ class AUBCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw<GfxFa
using CommandStreamReceiverSimulatedCommonHw<GfxFamily>::engineInfo;
using CommandStreamReceiverSimulatedCommonHw<GfxFamily>::stream;
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
void processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) override;

8
shared/source/command_stream/aub_command_stream_receiver_hw_base.inl
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2021 Intel Corporation
* Copyright (C) 2019-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -300,7 +300,7 @@ CommandStreamReceiver *AUBCommandStreamReceiverHw<GfxFamily>::create(const std::
}
template <typename GfxFamily>
bool AUBCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
SubmissionStatus AUBCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
if (subCaptureManager->isSubCaptureMode()) {
if (!subCaptureManager->isSubCaptureEnabled()) {
if (this->standalone) {
@@ -310,7 +310,7 @@ bool AUBCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer, Resi
pollAddress = ptrOffset(pollAddress, this->postSyncWriteOffset);
}
}
return true;
return SubmissionStatus::SUCCESS;
}
}
@@ -362,7 +362,7 @@ bool AUBCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer, Resi
}
getAubStream()->flush();
return true;
return SubmissionStatus::SUCCESS;
}
template <typename GfxFamily>

6
shared/source/command_stream/command_stream_receiver.cpp
View File

@@ -91,16 +91,16 @@ CommandStreamReceiver::~CommandStreamReceiver() {
getMemoryManager()->unregisterEngineForCsr(this);
}
int CommandStreamReceiver::submitBatchBuffer(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
SubmissionStatus CommandStreamReceiver::submitBatchBuffer(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
this->latestSentTaskCount = taskCount + 1;
auto flushed = this->flush(batchBuffer, allocationsForResidency);
SubmissionStatus retVal = this->flush(batchBuffer, allocationsForResidency);
if (!isUpdateTagFromWaitEnabled()) {
this->latestFlushedTaskCount = taskCount + 1;
}
taskCount++;
return !flushed;
return retVal;
}
void CommandStreamReceiver::makeResident(MultiGraphicsAllocation &gfxAllocation) {

7
shared/source/command_stream/command_stream_receiver.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -11,6 +11,7 @@
#include "shared/source/command_stream/csr_properties_flags.h"
#include "shared/source/command_stream/linear_stream.h"
#include "shared/source/command_stream/stream_properties.h"
#include "shared/source/command_stream/submission_status.h"
#include "shared/source/command_stream/submissions_aggregator.h"
#include "shared/source/command_stream/thread_arbitration_policy.h"
#include "shared/source/helpers/aligned_memory.h"
@@ -76,14 +77,14 @@ class CommandStreamReceiver {
const DeviceBitfield deviceBitfield);
virtual ~CommandStreamReceiver();
virtual bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) = 0;
virtual SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) = 0;
virtual CompletionStamp flushTask(LinearStream &commandStream, size_t commandStreamStart,
const IndirectHeap &dsh, const IndirectHeap &ioh, const IndirectHeap &ssh,
uint32_t taskLevel, DispatchFlags &dispatchFlags, Device &device) = 0;
virtual bool flushBatchedSubmissions() = 0;
MOCKABLE_VIRTUAL int submitBatchBuffer(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency);
MOCKABLE_VIRTUAL SubmissionStatus submitBatchBuffer(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency);
virtual void pollForCompletion() {}
virtual void programHardwareContext(LinearStream &cmdStream) = 0;
virtual size_t getCmdsSizeForHardwareContext() const = 0;

5
shared/source/command_stream/command_stream_receiver_hw.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -7,6 +7,7 @@
#pragma once
#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/command_stream/submission_status.h"
#include "shared/source/direct_submission/direct_submission_hw.h"
#include "shared/source/direct_submission/dispatchers/blitter_dispatcher.h"
#include "shared/source/direct_submission/dispatchers/render_dispatcher.h"
@@ -38,7 +39,7 @@ class CommandStreamReceiverHw : public CommandStreamReceiver {
const DeviceBitfield deviceBitfield);
~CommandStreamReceiverHw() override;
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
CompletionStamp flushTask(LinearStream &commandStream, size_t commandStreamStart,
const IndirectHeap &dsh, const IndirectHeap &ioh, const IndirectHeap &ssh,

8
shared/source/command_stream/command_stream_receiver_hw_base.inl
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2021 Intel Corporation
* Copyright (C) 2019-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -75,8 +75,8 @@ CommandStreamReceiverHw<GfxFamily>::CommandStreamReceiverHw(ExecutionEnvironment
}
template <typename GfxFamily>
bool CommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
return true;
SubmissionStatus CommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
return SubmissionStatus::SUCCESS;
}
template <typename GfxFamily>
@@ -754,7 +754,7 @@ inline bool CommandStreamReceiverHw<GfxFamily>::flushBatchedSubmissions() {
primaryCmdBuffer->batchBuffer.endCmdPtr = currentBBendLocation;
if (!this->flush(primaryCmdBuffer->batchBuffer, surfacesForSubmit)) {
if (this->flush(primaryCmdBuffer->batchBuffer, surfacesForSubmit) != SubmissionStatus::SUCCESS) {
submitResult = false;
break;
}

4
shared/source/command_stream/command_stream_receiver_with_aub_dump.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -26,7 +26,7 @@ class CommandStreamReceiverWithAUBDump : public BaseCSR {
CommandStreamReceiverWithAUBDump(const CommandStreamReceiverWithAUBDump &) = delete;
CommandStreamReceiverWithAUBDump &operator=(const CommandStreamReceiverWithAUBDump &) = delete;
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
void makeNonResident(GraphicsAllocation &gfxAllocation) override;
AubSubCaptureStatus checkAndActivateAubSubCapture(const std::string &kernelName) override;

4
shared/source/command_stream/command_stream_receiver_with_aub_dump.inl
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -32,7 +32,7 @@ CommandStreamReceiverWithAUBDump<BaseCSR>::CommandStreamReceiverWithAUBDump(cons
}
template <typename BaseCSR>
bool CommandStreamReceiverWithAUBDump<BaseCSR>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
SubmissionStatus CommandStreamReceiverWithAUBDump<BaseCSR>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
if (aubCSR) {
aubCSR->flush(batchBuffer, allocationsForResidency);
aubCSR->setLatestSentTaskCount(BaseCSR::peekLatestSentTaskCount());

21
shared/source/command_stream/submission_status.h Normal file
View File

@@ -0,0 +1,21 @@
/*
* Copyright (C) 2019-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include <cstdint>
namespace NEO {
enum class SubmissionStatus : uint32_t {
SUCCESS = 0,
FAILED,
OUT_OF_MEMORY,
UNSUPPORTED,
DEVICE_UNINITIALIZED,
};
} // namespace NEO

4
shared/source/command_stream/tbx_command_stream_receiver_hw.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -40,7 +40,7 @@ class TbxCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw<GfxFa
using CommandStreamReceiverSimulatedCommonHw<GfxFamily>::engineInfo;
using CommandStreamReceiverSimulatedCommonHw<GfxFamily>::stream;
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool useQuickKmdSleep, bool forcePowerSavingMode) override;
bool waitForCompletionWithTimeout(bool enableTimeout, int64_t timeoutMicroseconds, uint32_t taskCountToWait) override;

6
shared/source/command_stream/tbx_command_stream_receiver_hw.inl
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -206,7 +206,7 @@ CommandStreamReceiver *TbxCommandStreamReceiverHw<GfxFamily>::create(const std::
}
template <typename GfxFamily>
bool TbxCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
SubmissionStatus TbxCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
if (subCaptureManager) {
if (aubManager) {
aubManager->pause(false);
@@ -254,7 +254,7 @@ bool TbxCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer, Resi
subCaptureManager->disableSubCapture();
}
return true;
return SubmissionStatus::SUCCESS;
}
template <typename GfxFamily>

4
shared/source/direct_submission/direct_submission_hw.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -65,7 +65,7 @@ class DirectSubmissionHw {
bool startRingBuffer();
bool dispatchCommandBuffer(BatchBuffer &batchBuffer, FlushStampTracker &flushStamp);
MOCKABLE_VIRTUAL bool dispatchCommandBuffer(BatchBuffer &batchBuffer, FlushStampTracker &flushStamp);
static std::unique_ptr<DirectSubmissionHw<GfxFamily, Dispatcher>> create(Device &device, OsContext &osContext);

2
shared/source/memory_manager/memory_operations_status.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2020 Intel Corporation
* Copyright (C) 2019-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*

22
shared/source/os_interface/linux/drm_allocation.cpp
View File

@@ -68,27 +68,36 @@ bool DrmAllocation::setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regi
return true;
}
void DrmAllocation::makeBOsResident(OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind) {
int DrmAllocation::makeBOsResident(OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind) {
if (this->fragmentsStorage.fragmentCount) {
for (unsigned int f = 0; f < this->fragmentsStorage.fragmentCount; f++) {
if (!this->fragmentsStorage.fragmentStorageData[f].residency->resident[osContext->getContextId()]) {
bindBO(static_cast<OsHandleLinux *>(this->fragmentsStorage.fragmentStorageData[f].osHandleStorage)->bo, osContext, vmHandleId, bufferObjects, bind);
int retVal = bindBO(static_cast<OsHandleLinux *>(this->fragmentsStorage.fragmentStorageData[f].osHandleStorage)->bo, osContext, vmHandleId, bufferObjects, bind);
if (retVal) {
return retVal;
}
this->fragmentsStorage.fragmentStorageData[f].residency->resident[osContext->getContextId()] = true;
}
}
} else {
bindBOs(osContext, vmHandleId, bufferObjects, bind);
int retVal = bindBOs(osContext, vmHandleId, bufferObjects, bind);
if (retVal) {
return retVal;
}
}
return 0;
}
void DrmAllocation::bindBO(BufferObject *bo, OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind) {
int DrmAllocation::bindBO(BufferObject *bo, OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind) {
auto retVal = 0;
if (bo) {
bo->requireExplicitResidency(bo->peekDrm()->hasPageFaultSupport() && !shouldAllocationPageFault(bo->peekDrm()));
if (bufferObjects) {
if (bo->peekIsReusableAllocation()) {
for (auto bufferObject : *bufferObjects) {
if (bufferObject == bo) {
return;
return 0;
}
}
}
@@ -96,15 +105,14 @@ void DrmAllocation::bindBO(BufferObject *bo, OsContext *osContext, uint32_t vmHa
bufferObjects->push_back(bo);
} else {
auto retVal = 0;
if (bind) {
retVal = bo->bind(osContext, vmHandleId);
} else {
retVal = bo->unbind(osContext, vmHandleId);
}
UNRECOVERABLE_IF(retVal);
}
}
return retVal;
}
void DrmAllocation::registerBOBindExtHandle(Drm *drm) {

6
shared/source/os_interface/linux/drm_allocation.h
View File

@@ -91,9 +91,9 @@ class DrmAllocation : public GraphicsAllocation {
size_t getMmapSize() { return this->mmapSize; }
void setMmapSize(size_t size) { this->mmapSize = size; }
void makeBOsResident(OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind);
void bindBO(BufferObject *bo, OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind);
void bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind);
MOCKABLE_VIRTUAL int makeBOsResident(OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind);
MOCKABLE_VIRTUAL int bindBO(BufferObject *bo, OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind);
MOCKABLE_VIRTUAL int bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind);
MOCKABLE_VIRTUAL void registerBOBindExtHandle(Drm *drm);
void freeRegisteredBOBindExtHandles(Drm *drm);
void linkWithRegisteredHandle(uint32_t handle);

6
shared/source/os_interface/linux/drm_allocation_extended.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -13,9 +13,9 @@
namespace NEO {
void DrmAllocation::bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind) {
int DrmAllocation::bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vector<BufferObject *> *bufferObjects, bool bind) {
auto bo = this->getBO();
bindBO(bo, osContext, vmHandleId, bufferObjects, bind);
return bindBO(bo, osContext, vmHandleId, bufferObjects, bind);
}
bool DrmAllocation::setCacheRegion(Drm *drm, CacheRegion regionIndex) {

4
shared/source/os_interface/linux/drm_command_stream.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -44,7 +44,7 @@ class DrmCommandStreamReceiver : public DeviceCommandStreamReceiver<GfxFamily> {
const DeviceBitfield deviceBitfield,
gemCloseWorkerMode mode = gemCloseWorkerMode::gemCloseWorkerActive);
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
MOCKABLE_VIRTUAL void processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) override;
void makeNonResident(GraphicsAllocation &gfxAllocation) override;
bool waitForFlushStamp(FlushStamp &flushStampToWait) override;

38
shared/source/os_interface/linux/drm_command_stream.inl
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -77,14 +77,14 @@ DrmCommandStreamReceiver<GfxFamily>::DrmCommandStreamReceiver(ExecutionEnvironme
}
template <typename GfxFamily>
bool DrmCommandStreamReceiver<GfxFamily>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
SubmissionStatus DrmCommandStreamReceiver<GfxFamily>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
this->printDeviceIndex();
DrmAllocation *alloc = static_cast<DrmAllocation *>(batchBuffer.commandBufferAllocation);
DEBUG_BREAK_IF(!alloc);
BufferObject *bb = alloc->getBO();
if (bb == nullptr) {
return false;
return SubmissionStatus::OUT_OF_MEMORY;
}
if (this->lastSentSliceCount != batchBuffer.sliceCount) {
@@ -102,19 +102,39 @@ bool DrmCommandStreamReceiver<GfxFamily>::flush(BatchBuffer &batchBuffer, Reside
this->printBOsForSubmit(allocationsForResidency, *batchBuffer.commandBufferAllocation);
memoryOperationsInterface->mergeWithResidencyContainer(this->osContext, allocationsForResidency);
MemoryOperationsStatus retVal = memoryOperationsInterface->mergeWithResidencyContainer(this->osContext, allocationsForResidency);
if (retVal != MemoryOperationsStatus::SUCCESS) {
if (retVal == MemoryOperationsStatus::OUT_OF_MEMORY) {
return SubmissionStatus::OUT_OF_MEMORY;
}
return SubmissionStatus::FAILED;
}
if (this->drm->isVmBindAvailable()) {
memoryOperationsInterface->makeResidentWithinOsContext(this->osContext, ArrayRef<GraphicsAllocation *>(&batchBuffer.commandBufferAllocation, 1), true);
retVal = memoryOperationsInterface->makeResidentWithinOsContext(this->osContext, ArrayRef<GraphicsAllocation *>(&batchBuffer.commandBufferAllocation, 1), true);
if (retVal != MemoryOperationsStatus::SUCCESS) {
if (retVal == MemoryOperationsStatus::OUT_OF_MEMORY) {
return SubmissionStatus::OUT_OF_MEMORY;
}
return SubmissionStatus::FAILED;
}
}
if (this->directSubmission.get()) {
this->startControllingDirectSubmissions();
return this->directSubmission->dispatchCommandBuffer(batchBuffer, *this->flushStamp.get());
bool ret = this->directSubmission->dispatchCommandBuffer(batchBuffer, *this->flushStamp.get());
if (ret == false) {
return SubmissionStatus::FAILED;
}
return SubmissionStatus::SUCCESS;
}
if (this->blitterDirectSubmission.get()) {
this->startControllingDirectSubmissions();
return this->blitterDirectSubmission->dispatchCommandBuffer(batchBuffer, *this->flushStamp.get());
bool ret = this->blitterDirectSubmission->dispatchCommandBuffer(batchBuffer, *this->flushStamp.get());
if (ret == false) {
return SubmissionStatus::FAILED;
}
return SubmissionStatus::SUCCESS;
}
if (isUserFenceWaitActive()) {
@@ -130,10 +150,10 @@ bool DrmCommandStreamReceiver<GfxFamily>::flush(BatchBuffer &batchBuffer, Reside
}
if (ret) {
return false;
return SubmissionStatus::FAILED;
}
return true;
return SubmissionStatus::SUCCESS;
}
template <typename GfxFamily>

4
shared/source/os_interface/linux/drm_memory_operations_handler.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2021 Intel Corporation
* Copyright (C) 2019-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -20,7 +20,7 @@ class DrmMemoryOperationsHandler : public MemoryOperationsHandler {
DrmMemoryOperationsHandler() = default;
~DrmMemoryOperationsHandler() override = default;
virtual void mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) = 0;
virtual MemoryOperationsStatus mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) = 0;
virtual std::unique_lock<std::mutex> lockHandlerIfUsed() = 0;
virtual void evictUnusedAllocations(bool waitForCompletion, bool isLockNeeded) = 0;

29
shared/source/os_interface/linux/drm_memory_operations_handler_bind.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -40,7 +40,10 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResidentWithinOsConte
auto drmAllocation = static_cast<DrmAllocation *>(*gfxAllocation);
for (auto drmIterator = 0u; drmIterator < osContext->getDeviceBitfield().size(); drmIterator++) {
if (osContext->getDeviceBitfield().test(drmIterator)) {
drmAllocation->makeBOsResident(osContext, drmIterator, nullptr, true);
int result = drmAllocation->makeBOsResident(osContext, drmIterator, nullptr, true);
if (result) {
return MemoryOperationsStatus::OUT_OF_MEMORY;
}
}
}
if (!evictable) {
@@ -64,18 +67,26 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerBind::evict(Device *device, Gra
MemoryOperationsStatus DrmMemoryOperationsHandlerBind::evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) {
std::lock_guard<std::mutex> lock(mutex);
evictImpl(osContext, gfxAllocation, osContext->getDeviceBitfield());
int retVal = evictImpl(osContext, gfxAllocation, osContext->getDeviceBitfield());
if (retVal) {
return MemoryOperationsStatus::FAILED;
}
return MemoryOperationsStatus::SUCCESS;
}
void DrmMemoryOperationsHandlerBind::evictImpl(OsContext *osContext, GraphicsAllocation &gfxAllocation, DeviceBitfield deviceBitfield) {
int DrmMemoryOperationsHandlerBind::evictImpl(OsContext *osContext, GraphicsAllocation &gfxAllocation, DeviceBitfield deviceBitfield) {
auto drmAllocation = static_cast<DrmAllocation *>(&gfxAllocation);
for (auto drmIterator = 0u; drmIterator < deviceBitfield.size(); drmIterator++) {
if (deviceBitfield.test(drmIterator)) {
drmAllocation->makeBOsResident(osContext, drmIterator, nullptr, false);
int retVal = drmAllocation->makeBOsResident(osContext, drmIterator, nullptr, false);
if (retVal) {
return retVal;
}
}
}
drmAllocation->updateResidencyTaskCount(GraphicsAllocation::objectNotResident, osContext->getContextId());
return 0;
}
MemoryOperationsStatus DrmMemoryOperationsHandlerBind::isResident(Device *device, GraphicsAllocation &gfxAllocation) {
@@ -92,8 +103,11 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerBind::isResident(Device *device
return MemoryOperationsStatus::MEMORY_NOT_FOUND;
}
void DrmMemoryOperationsHandlerBind::mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) {
this->makeResidentWithinOsContext(osContext, ArrayRef<GraphicsAllocation *>(residencyContainer), true);
MemoryOperationsStatus DrmMemoryOperationsHandlerBind::mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) {
MemoryOperationsStatus retVal = this->makeResidentWithinOsContext(osContext, ArrayRef<GraphicsAllocation *>(residencyContainer), true);
if (retVal != MemoryOperationsStatus::SUCCESS) {
return retVal;
}
auto clearContainer = true;
@@ -104,6 +118,7 @@ void DrmMemoryOperationsHandlerBind::mergeWithResidencyContainer(OsContext *osCo
if (clearContainer) {
residencyContainer.clear();
}
return MemoryOperationsStatus::SUCCESS;
}
std::unique_lock<std::mutex> DrmMemoryOperationsHandlerBind::lockHandlerIfUsed() {

6
shared/source/os_interface/linux/drm_memory_operations_handler_bind.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -22,13 +22,13 @@ class DrmMemoryOperationsHandlerBind : public DrmMemoryOperationsHandler {
MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override;
void mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) override;
MemoryOperationsStatus mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) override;
std::unique_lock<std::mutex> lockHandlerIfUsed() override;
void evictUnusedAllocations(bool waitForCompletion, bool isLockNeeded) override;
protected:
void evictImpl(OsContext *osContext, GraphicsAllocation &gfxAllocation, DeviceBitfield deviceBitfield);
MOCKABLE_VIRTUAL int evictImpl(OsContext *osContext, GraphicsAllocation &gfxAllocation, DeviceBitfield deviceBitfield);
void evictUnusedAllocationsImpl(std::vector<GraphicsAllocation *> &allocationsForEviction, bool waitForCompletion);
RootDeviceEnvironment &rootDeviceEnvironment;

5
shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -47,13 +47,14 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::isResident(Device *dev
return MemoryOperationsStatus::SUCCESS;
}
void DrmMemoryOperationsHandlerDefault::mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) {
MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) {
for (auto gfxAllocation = this->residency.begin(); gfxAllocation != this->residency.end(); gfxAllocation++) {
auto ret = std::find(residencyContainer.begin(), residencyContainer.end(), *gfxAllocation);
if (ret == residencyContainer.end()) {
residencyContainer.push_back(*gfxAllocation);
}
}
return MemoryOperationsStatus::SUCCESS;
}
std::unique_lock<std::mutex> DrmMemoryOperationsHandlerDefault::lockHandlerIfUsed() {

4
shared/source/os_interface/linux/drm_memory_operations_handler_default.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -23,7 +23,7 @@ class DrmMemoryOperationsHandlerDefault : public DrmMemoryOperationsHandler {
MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override;
void mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) override;
MemoryOperationsStatus mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) override;
std::unique_lock<std::mutex> lockHandlerIfUsed() override;
void evictUnusedAllocations(bool waitForCompletion, bool isLockNeeded) override;

5
shared/source/os_interface/windows/wddm_device_command_stream.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -7,6 +7,7 @@
#pragma once
#include "shared/source/command_stream/device_command_stream.h"
#include "shared/source/command_stream/submission_status.h"
struct COMMAND_BUFFER_HEADER_REC;
@@ -24,7 +25,7 @@ class WddmCommandStreamReceiver : public DeviceCommandStreamReceiver<GfxFamily>
WddmCommandStreamReceiver(ExecutionEnvironment &executionEnvironment, uint32_t rootDeviceIndex, const DeviceBitfield deviceBitfield);
virtual ~WddmCommandStreamReceiver();
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
void processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) override;
void processEviction() override;
bool waitForFlushStamp(FlushStamp &flushStampToWait) override;

22
shared/source/os_interface/windows/wddm_device_command_stream.inl
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -66,7 +66,7 @@ WddmCommandStreamReceiver<GfxFamily>::~WddmCommandStreamReceiver() {
}
template <typename GfxFamily>
bool WddmCommandStreamReceiver<GfxFamily>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
SubmissionStatus WddmCommandStreamReceiver<GfxFamily>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
this->printDeviceIndex();
auto commandStreamAddress = ptrOffset(batchBuffer.commandBufferAllocation->getGpuAddress(), batchBuffer.startOffset);
@@ -75,10 +75,18 @@ bool WddmCommandStreamReceiver<GfxFamily>::flush(BatchBuffer &batchBuffer, Resid
perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "Wddm CSR processing residency set: %zu\n", allocationsForResidency.size());
this->processResidency(allocationsForResidency, 0u);
if (this->directSubmission.get()) {
return this->directSubmission->dispatchCommandBuffer(batchBuffer, *(this->flushStamp.get()));
bool ret = this->directSubmission->dispatchCommandBuffer(batchBuffer, *(this->flushStamp.get()));
if (ret == false) {
return SubmissionStatus::FAILED;
}
return SubmissionStatus::SUCCESS;
}
if (this->blitterDirectSubmission.get()) {
return this->blitterDirectSubmission->dispatchCommandBuffer(batchBuffer, *(this->flushStamp.get()));
bool ret = this->blitterDirectSubmission->dispatchCommandBuffer(batchBuffer, *(this->flushStamp.get()));
if (ret == false) {
return SubmissionStatus::FAILED;
}
return SubmissionStatus::SUCCESS;
}
COMMAND_BUFFER_HEADER *pHeader = reinterpret_cast<COMMAND_BUFFER_HEADER *>(commandBufferHeader);
@@ -110,7 +118,11 @@ bool WddmCommandStreamReceiver<GfxFamily>::flush(BatchBuffer &batchBuffer, Resid
auto status = wddm->submit(commandStreamAddress, batchBuffer.usedSize - batchBuffer.startOffset, commandBufferHeader, submitArgs);
this->flushStamp->setStamp(submitArgs.monitorFence->lastSubmittedFence);
return status;
if (status == false) {
return SubmissionStatus::FAILED;
}
return SubmissionStatus::SUCCESS;
}
template <typename GfxFamily>

4
shared/test/common/libult/ult_command_stream_receiver.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -140,7 +140,7 @@ class UltCommandStreamReceiver : public CommandStreamReceiverHw<GfxFamily>, publ
BaseClass::makeSurfacePackNonResident(allocationsForResidency);
}
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
NEO::SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
if (recordFlusheBatchBuffer) {
latestFlushedBatchBuffer = batchBuffer;
}

6
shared/test/common/mocks/mock_command_stream_receiver.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2021 Intel Corporation
* Copyright (C) 2021-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -9,8 +9,8 @@
volatile uint32_t MockCommandStreamReceiver::mockTagAddress[MockCommandStreamReceiver::tagSize];
bool MockCommandStreamReceiver::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
return true;
SubmissionStatus MockCommandStreamReceiver::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
return SubmissionStatus::SUCCESS;
}
CompletionStamp MockCommandStreamReceiver::flushTask(

21
shared/test/common/mocks/mock_command_stream_receiver.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -54,7 +54,7 @@ class MockCommandStreamReceiver : public CommandStreamReceiver {
waitForCompletionWithTimeoutCalled++;
return true;
}
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
void flushTagUpdate() override{};
void flushNonKernelTask(GraphicsAllocation *eventAlloc, uint64_t immediateGpuAddress, uint64_t immediateData, PipeControlArgs &args, bool isWaitOnEvents, bool startOfDispatch, bool endOfDispatch) override{};
@@ -152,8 +152,17 @@ class MockCommandStreamReceiverWithFailingSubmitBatch : public MockCommandStream
public:
MockCommandStreamReceiverWithFailingSubmitBatch(ExecutionEnvironment &executionEnvironment, uint32_t rootDeviceIndex, const DeviceBitfield deviceBitfield)
: MockCommandStreamReceiver(executionEnvironment, rootDeviceIndex, deviceBitfield) {}
int submitBatchBuffer(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
return -1;
SubmissionStatus submitBatchBuffer(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
return SubmissionStatus::FAILED;
}
};
class MockCommandStreamReceiverWithOutOfMemorySubmitBatch : public MockCommandStreamReceiver {
public:
MockCommandStreamReceiverWithOutOfMemorySubmitBatch(ExecutionEnvironment &executionEnvironment, uint32_t rootDeviceIndex, const DeviceBitfield deviceBitfield)
: MockCommandStreamReceiver(executionEnvironment, rootDeviceIndex, deviceBitfield) {}
SubmissionStatus submitBatchBuffer(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
return SubmissionStatus::OUT_OF_MEMORY;
}
};
@@ -204,14 +213,14 @@ class MockCsrHw2 : public CommandStreamReceiverHw<GfxFamily> {
bool peekMediaVfeStateDirty() const { return mediaVfeStateDirty; }
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
flushCalledCount++;
if (recordedCommandBuffer) {
recordedCommandBuffer->batchBuffer = batchBuffer;
}
copyOfAllocations = allocationsForResidency;
flushStamp->setStamp(flushStamp->peekStamp() + 1);
return true;
return SubmissionStatus::SUCCESS;
}
CompletionStamp flushTask(LinearStream &commandStream, size_t commandStreamStart,

6
shared/test/common/mocks/mock_csr.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2021 Intel Corporation
* Copyright (C) 2018-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -109,8 +109,8 @@ class MockCsr : public MockCsrBase<GfxFamily> {
: BaseClass(execStamp, executionEnvironment, rootDeviceIndex, deviceBitfield) {
}
bool flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
return true;
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
return SubmissionStatus::SUCCESS;
}
CompletionStamp flushTask(