Media
/
compute-runtime
mirror of https://github.com/intel/compute-runtime.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allocate and consume shared heaps atomically 

Related-To: NEO-5055

Signed-off-by: Zbigniew Zdanowicz <zbigniew.zdanowicz@intel.com>
This commit is contained in:
Zbigniew Zdanowicz 2023-02-02 18:57:24 +00:00 committed by Compute-Runtime-Automation
parent ee6eb70f1a
commit f2be0ebfc4
11 changed files with 437 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
level_zero/core/source/cmdlist/cmdlist_hw.h
View File

@ -324,6 +324,7 @@ struct CommandListCoreFamily : CommandListImp {
bool containsAnyKernel = false;
bool pipeControlMultiKernelEventSync = false;
bool compactL3FlushEventPacket = false;
bool dynamicHeapRequired = false;
};
template <PRODUCT_FAMILY gfxProductFamily>

1
level_zero/core/source/cmdlist/cmdlist_hw.inl
View File

@ -155,6 +155,7 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::initialize(Device *device, NEO
this->pipeControlMultiKernelEventSync = L0GfxCoreHelper::usePipeControlMultiKernelEventSync(hwInfo);
this->compactL3FlushEventPacket = L0GfxCoreHelper::useCompactL3FlushEventPacket(hwInfo);
this->signalAllEventPackets = L0GfxCoreHelper::useSignalAllEventPackets(hwInfo);
this->dynamicHeapRequired = NEO::EncodeDispatchKernel<GfxFamily>::isDshNeeded(device->getDeviceInfo());
if (device->isImplicitScalingCapable() && !this->internalUsage && !isCopyOnly()) {
this->partitionCount = static_cast<uint32_t>(this->device->getNEODevice()->getDeviceBitfield().count());

23
level_zero/core/source/cmdlist/cmdlist_hw_base.inl
View File

@ -53,15 +53,26 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelWithParams(K
if (this->immediateCmdListHeapSharing || this->stateBaseAddressTracking) {
auto kernelInfo = kernelImmutableData->getKernelInfo();
commandContainer.ensureHeapSizePrepared(
auto &sshReserveConfig = commandContainer.getSurfaceStateHeapReserve();
NEO::HeapReserveArguments sshReserveArgs = {
sshReserveConfig.indirectHeapReservation,
NEO::EncodeDispatchKernel<GfxFamily>::getSizeRequiredSsh(*kernelInfo),
NEO::EncodeDispatchKernel<GfxFamily>::getDefaultSshAlignment(),
NEO::EncodeDispatchKernel<GfxFamily>::getSizeRequiredDsh(kernelDescriptor, commandContainer.getNumIddPerBlock()),
NEO::EncodeDispatchKernel<GfxFamily>::getDefaultDshAlignment(), true);
NEO::EncodeDispatchKernel<GfxFamily>::getDefaultSshAlignment()};
ssh = commandContainer.getIndirectHeap(NEO::HeapType::SURFACE_STATE);
dsh = commandContainer.getIndirectHeap(NEO::HeapType::DYNAMIC_STATE);
auto &dshReserveConfig = commandContainer.getDynamicStateHeapReserve();
NEO::HeapReserveArguments dshReserveArgs = {
dshReserveConfig.indirectHeapReservation,
NEO::EncodeDispatchKernel<GfxFamily>::getSizeRequiredDsh(kernelDescriptor, commandContainer.getNumIddPerBlock()),
NEO::EncodeDispatchKernel<GfxFamily>::getDefaultDshAlignment()};
commandContainer.reserveSpaceForDispatch(
sshReserveArgs,
dshReserveArgs, true);
ssh = sshReserveArgs.indirectHeapReservation;
dsh = dshReserveArgs.indirectHeapReservation;
}
appendEventForProfiling(event, true);
auto perThreadScratchSize = std::max<std::uint32_t>(this->getCommandListPerThreadScratchSize(),
kernel->getImmutableData()->getDescriptor().kernelAttributes.perThreadScratchSize[0]);

15
level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl
View File

@ -129,8 +129,19 @@ NEO::CompletionStamp CommandListCoreFamilyImmediate<gfxCoreFamily>::flushRegular
NEO::IndirectHeap *ssh = nullptr;
if (!NEO::ApiSpecificConfig::getBindlessConfiguration()) {
dsh = (this->commandContainer.getIndirectHeap(NEO::IndirectHeap::Type::DYNAMIC_STATE));
ssh = (this->commandContainer.getIndirectHeap(NEO::IndirectHeap::Type::SURFACE_STATE));
if (this->immediateCmdListHeapSharing) {
auto &sshReserveConfig = this->commandContainer.getSurfaceStateHeapReserve();
if (sshReserveConfig.indirectHeapReservation->getGraphicsAllocation()) {
ssh = sshReserveConfig.indirectHeapReservation;
}
auto &dshReserveConfig = this->commandContainer.getDynamicStateHeapReserve();
if (this->dynamicHeapRequired && dshReserveConfig.indirectHeapReservation->getGraphicsAllocation()) {
dsh = dshReserveConfig.indirectHeapReservation;
}
} else {
dsh = this->commandContainer.getIndirectHeap(NEO::IndirectHeap::Type::DYNAMIC_STATE);
ssh = this->commandContainer.getIndirectHeap(NEO::IndirectHeap::Type::SURFACE_STATE);
}
} else {
dsh = this->device->getNEODevice()->getBindlessHeapsHelper()->getHeap(NEO::BindlessHeapsHelper::BindlesHeapType::GLOBAL_DSH);
ssh = this->device->getNEODevice()->getBindlessHeapsHelper()->getHeap(NEO::BindlessHeapsHelper::BindlesHeapType::GLOBAL_SSH);

26
level_zero/core/source/cmdlist/cmdlist_hw_xehp_and_later.inl
View File

@ -142,19 +142,29 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelWithParams(K
NEO::IndirectHeap *dsh = nullptr;
if (this->immediateCmdListHeapSharing || this->stateBaseAddressTracking) {
const bool getDsh = NEO::EncodeDispatchKernel<GfxFamily>::isDshNeeded(device->getDeviceInfo());
auto kernelInfo = kernelImmutableData->getKernelInfo();
commandContainer.ensureHeapSizePrepared(
auto &sshReserveConfig = commandContainer.getSurfaceStateHeapReserve();
NEO::HeapReserveArguments sshReserveArgs = {
sshReserveConfig.indirectHeapReservation,
NEO::EncodeDispatchKernel<GfxFamily>::getSizeRequiredSsh(*kernelInfo),
NEO::EncodeDispatchKernel<GfxFamily>::getDefaultSshAlignment(),
NEO::EncodeDispatchKernel<GfxFamily>::getSizeRequiredDsh(kernelDescriptor, 0),
NEO::EncodeDispatchKernel<GfxFamily>::getDefaultDshAlignment(), getDsh);
NEO::EncodeDispatchKernel<GfxFamily>::getDefaultSshAlignment()};
ssh = commandContainer.getIndirectHeap(NEO::HeapType::SURFACE_STATE);
if (getDsh) {
dsh = commandContainer.getIndirectHeap(NEO::HeapType::DYNAMIC_STATE);
NEO::HeapReserveArguments dshReserveArgs = {};
if (this->dynamicHeapRequired) {
auto &dshReserveConfig = commandContainer.getDynamicStateHeapReserve();
dshReserveArgs = {
dshReserveConfig.indirectHeapReservation,
NEO::EncodeDispatchKernel<GfxFamily>::getSizeRequiredDsh(kernelDescriptor, 0),
NEO::EncodeDispatchKernel<GfxFamily>::getDefaultDshAlignment()};
}
commandContainer.reserveSpaceForDispatch(
sshReserveArgs,
dshReserveArgs, this->dynamicHeapRequired);
ssh = sshReserveArgs.indirectHeapReservation;
dsh = dshReserveArgs.indirectHeapReservation;
}
commandListPerThreadScratchSize = std::max<uint32_t>(commandListPerThreadScratchSize, kernelDescriptor.kernelAttributes.perThreadScratchSize[0]);
commandListPerThreadPrivateScratchSize = std::max<uint32_t>(commandListPerThreadPrivateScratchSize, kernelDescriptor.kernelAttributes.perThreadScratchSize[1]);

3
level_zero/core/source/cmdlist/cmdlist_imp.cpp
View File

@ -143,7 +143,8 @@ CommandList *CommandList::createImmediate(uint32_t productFamily, Device *device
PRINT_DEBUG_STRING(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "Flush Task for Immediate command list : %s\n", commandList->isFlushTaskSubmissionEnabled ? "Enabled" : "Disabled");
auto &rootDeviceEnvironment = device->getNEODevice()->getRootDeviceEnvironment();
commandList->immediateCmdListHeapSharing = L0GfxCoreHelper::enableImmediateCmdListHeapSharing(rootDeviceEnvironment, commandList->isFlushTaskSubmissionEnabled);
bool enabledCmdListSharing = !NEO::EngineHelper::isCopyOnlyEngineType(engineGroupType) && commandList->isFlushTaskSubmissionEnabled;
commandList->immediateCmdListHeapSharing = L0GfxCoreHelper::enableImmediateCmdListHeapSharing(rootDeviceEnvironment, enabledCmdListSharing);
}
returnValue = commandList->initialize(device, engineGroupType, desc->flags);
if (returnValue != ZE_RESULT_SUCCESS) {

14
level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_6.cpp
View File

@ -930,7 +930,17 @@ HWTEST2_F(ImmediateCmdListSharedHeapsTest, givenMultipleCommandListsUsingSharedH
auto containerDshHeapCoexisting = cmdContainerCoexisting.getIndirectHeap(HeapType::DYNAMIC_STATE);
auto containerSshHeapCoexisting = cmdContainerCoexisting.getIndirectHeap(HeapType::SURFACE_STATE);
size_t dshAlignment = NEO::EncodeDispatchKernel<FamilyType>::getDefaultDshAlignment();
size_t sshAlignment = NEO::EncodeDispatchKernel<FamilyType>::getDefaultSshAlignment();
void *ptr = containerSshHeapCoexisting->getSpace(0);
size_t expectedSshAlignedSize = sshEstimated + ptrDiff(alignUp(ptr, sshAlignment), ptr);
size_t expectedDshAlignedSize = dshEstimated;
if (dshPresent) {
ptr = containerDshHeapCoexisting->getSpace(0);
expectedDshAlignedSize += ptrDiff(alignUp(ptr, dshAlignment), ptr);
EXPECT_EQ(csrDshHeap, containerDshHeapCoexisting);
} else {
EXPECT_EQ(nullptr, containerDshHeapCoexisting);
@ -957,8 +967,8 @@ HWTEST2_F(ImmediateCmdListSharedHeapsTest, givenMultipleCommandListsUsingSharedH
}
EXPECT_LT(0u, sshUsed);
EXPECT_GE(dshEstimated, dshUsed);
EXPECT_GE(sshEstimated, sshUsed);
EXPECT_GE(expectedDshAlignedSize, dshUsed);
EXPECT_GE(expectedSshAlignedSize, sshUsed);
}
} // namespace ult

49
shared/source/command_container/cmdcontainer.cpp
View File

@ -331,28 +331,51 @@ IndirectHeap *CommandContainer::getIndirectHeap(HeapType heapType) {
}
}
IndirectHeap *CommandContainer::getCsrAlignedSize(HeapType heapType, size_t size, size_t alignment) {
void *ptr = immediateCmdListCsr->getIndirectHeapCurrentPtr(heapType);
size_t totalSize = size + ptrDiff(alignUp(ptr, alignment), ptr);
IndirectHeap *CommandContainer::initIndirectHeapReservation(ReservedIndirectHeap *indirectHeapReservation, size_t size, size_t alignment, HeapType heapType) {
void *currentHeap = immediateCmdListCsr->getIndirectHeapCurrentPtr(heapType);
auto totalRequiredSize = size + ptrDiff(alignUp(currentHeap, alignment), currentHeap);
auto baseHeap = &immediateCmdListCsr->getIndirectHeap(heapType, totalSize);
auto baseHeap = &immediateCmdListCsr->getIndirectHeap(heapType, totalRequiredSize);
auto usedSize = baseHeap->getUsed();
void *heapCpuBase = baseHeap->getCpuBase();
auto consumedSize = usedSize + totalRequiredSize;
baseHeap->getSpace(totalRequiredSize);
indirectHeapReservation->replaceGraphicsAllocation(baseHeap->getGraphicsAllocation());
indirectHeapReservation->replaceBuffer(heapCpuBase, consumedSize);
indirectHeapReservation->getSpace(usedSize);
indirectHeapReservation->setHeapSizeInPages(baseHeap->getHeapSizeInPages());
return baseHeap;
}
void CommandContainer::ensureHeapSizePrepared(size_t sshRequiredSize, size_t sshDefaultAlignment, size_t dshRequiredSize, size_t dshDefaultAlignment, bool getDsh) {
void CommandContainer::reserveSpaceForDispatch(HeapReserveArguments &sshReserveArg, HeapReserveArguments &dshReserveArg, bool getDsh) {
size_t sshAlignment = sshReserveArg.alignment;
size_t dshAlignment = dshReserveArg.alignment;
if (sshReserveArg.size == 0) {
sshAlignment = 1;
}
if (dshReserveArg.size == 0) {
dshAlignment = 1;
}
if (immediateCmdListCsr) {
auto lock = immediateCmdListCsr->obtainUniqueOwnership();
sharedSshCsrHeap = getCsrAlignedSize(HeapType::SURFACE_STATE, sshRequiredSize, sshDefaultAlignment);
sharedSshCsrHeap = this->initIndirectHeapReservation(sshReserveArg.indirectHeapReservation, sshReserveArg.size, sshAlignment, HeapType::SURFACE_STATE);
if (getDsh) {
sharedDshCsrHeap = getCsrAlignedSize(HeapType::DYNAMIC_STATE, dshRequiredSize, dshDefaultAlignment);
sharedDshCsrHeap = this->initIndirectHeapReservation(dshReserveArg.indirectHeapReservation, dshReserveArg.size, dshAlignment, HeapType::DYNAMIC_STATE);
}
} else {
this->getHeapWithRequiredSizeAndAlignment(HeapType::SURFACE_STATE, sshRequiredSize, sshDefaultAlignment);
this->getHeapWithRequiredSizeAndAlignment(HeapType::SURFACE_STATE, sshReserveArg.size, sshAlignment);
if (getDsh) {
this->getHeapWithRequiredSizeAndAlignment(HeapType::DYNAMIC_STATE, dshRequiredSize, dshDefaultAlignment);
this->getHeapWithRequiredSizeAndAlignment(HeapType::DYNAMIC_STATE, dshReserveArg.size, dshAlignment);
}
// private heaps can be accessed directly
sshReserveArg.indirectHeapReservation = nullptr;
dshReserveArg.indirectHeapReservation = nullptr;
}
}
@ -455,4 +478,12 @@ void CommandContainer::storeAllocationAndFlushTagUpdate(GraphicsAllocation *allo
this->immediateCmdListCsr->flushTagUpdate();
}
HeapReserveData::HeapReserveData() {
object = std::make_unique<NEO::ReservedIndirectHeap>(nullptr, false);
indirectHeapReservation = object.get();
}
HeapReserveData::~HeapReserveData() {
}
} // namespace NEO

36
shared/source/command_container/cmdcontainer.h
View File

@ -16,13 +16,14 @@
#include <vector>
namespace NEO {
class HeapHelper;
class AllocationsList;
class CommandStreamReceiver;
class Device;
class GraphicsAllocation;
class LinearStream;
class AllocationsList;
class HeapHelper;
class IndirectHeap;
class LinearStream;
class ReservedIndirectHeap;
using ResidencyContainer = std::vector<GraphicsAllocation *>;
using CmdBufferContainer = std::vector<GraphicsAllocation *>;
@ -39,6 +40,21 @@ inline constexpr auto minCommandQueueCommandStreamSize = 2 * MemoryConstants::ca
inline constexpr auto csOverfetchSize = MemoryConstants::pageSize;
} // namespace CSRequirements
struct HeapReserveData {
HeapReserveData();
virtual ~HeapReserveData();
ReservedIndirectHeap *indirectHeapReservation = nullptr;
protected:
std::unique_ptr<ReservedIndirectHeap> object;
};
struct HeapReserveArguments {
ReservedIndirectHeap *indirectHeapReservation = nullptr;
size_t size = 0;
size_t alignment = 0;
};
class CommandContainer : public NonCopyableOrMovableClass {
public:
enum class ErrorCode {
@ -119,7 +135,8 @@ class CommandContainer : public NonCopyableOrMovableClass {
bool immediateCmdListSharedHeap(HeapType heapType) {
return (heapSharingEnabled && (heapType == HeapType::DYNAMIC_STATE || heapType == HeapType::SURFACE_STATE));
}
void ensureHeapSizePrepared(size_t sshRequiredSize, size_t sshDefaultAlignment, size_t dshRequiredSize, size_t dshDefaultAlignment, bool getDsh);
void reserveSpaceForDispatch(HeapReserveArguments &sshReserveArg, HeapReserveArguments &dshReserveArg, bool getDsh);
GraphicsAllocation *reuseExistingCmdBuffer();
GraphicsAllocation *allocateCommandBuffer();
@ -129,6 +146,13 @@ class CommandContainer : public NonCopyableOrMovableClass {
void fillReusableAllocationLists();
void storeAllocationAndFlushTagUpdate(GraphicsAllocation *allocation);
HeapReserveData &getSurfaceStateHeapReserve() {
return surfaceStateHeapReserveData;
}
HeapReserveData &getDynamicStateHeapReserve() {
return dynamicStateHeapReserveData;
}
HeapContainer sshAllocations;
uint64_t currentLinearStreamStartOffset = 0u;
uint32_t slmSize = std::numeric_limits<uint32_t>::max();
@ -141,9 +165,11 @@ class CommandContainer : public NonCopyableOrMovableClass {
size_t getTotalCmdBufferSize();
IndirectHeap *getHeapWithRequiredSize(HeapType heapType, size_t sizeRequired, size_t alignment, bool allowGrow);
void createAndAssignNewHeap(HeapType heapType, size_t size);
IndirectHeap *getCsrAlignedSize(HeapType heapType, size_t size, size_t alignment);
IndirectHeap *initIndirectHeapReservation(ReservedIndirectHeap *indirectHeapReservation, size_t size, size_t alignment, HeapType heapType);
GraphicsAllocation *allocationIndirectHeaps[HeapType::NUM_TYPES] = {};
std::unique_ptr<IndirectHeap> indirectHeaps[HeapType::NUM_TYPES];
HeapReserveData dynamicStateHeapReserveData;
HeapReserveData surfaceStateHeapReserveData;
CmdBufferContainer cmdBufferAllocations;
ResidencyContainer residencyContainer;

29
shared/source/indirect_heap/indirect_heap.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2022 Intel Corporation
* Copyright (C) 2019-2023 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -30,7 +30,7 @@ class IndirectHeap : public LinearStream {
public:
using Type = IndirectHeapType;
IndirectHeap(void *graphicsAllocation, size_t bufferSize) : BaseClass(graphicsAllocation, bufferSize){};
IndirectHeap(void *buffer, size_t bufferSize) : BaseClass(buffer, bufferSize){};
IndirectHeap(GraphicsAllocation *graphicsAllocation) : BaseClass(graphicsAllocation) {}
IndirectHeap(GraphicsAllocation *graphicsAllocation, bool canBeUtilizedAs4GbHeap)
: BaseClass(graphicsAllocation), canBeUtilizedAs4GbHeap(canBeUtilizedAs4GbHeap) {}
@ -42,7 +42,7 @@ class IndirectHeap : public LinearStream {
void align(size_t alignment);
uint64_t getHeapGpuStartOffset() const;
uint64_t getHeapGpuBase() const;
uint32_t getHeapSizeInPages() const;
virtual uint32_t getHeapSizeInPages() const;
protected:
bool canBeUtilizedAs4GbHeap = false;
@ -76,4 +76,27 @@ inline uint64_t IndirectHeap::getHeapGpuBase() const {
return this->graphicsAllocation->getGpuAddress();
}
}
class ReservedIndirectHeap : public IndirectHeap {
public:
ReservedIndirectHeap(void *buffer, size_t bufferSize) : IndirectHeap(buffer, bufferSize) {}
ReservedIndirectHeap(GraphicsAllocation *graphicsAllocation) : IndirectHeap(graphicsAllocation) {}
ReservedIndirectHeap(GraphicsAllocation *graphicsAllocation, bool canBeUtilizedAs4GbHeap)
: IndirectHeap(graphicsAllocation, canBeUtilizedAs4GbHeap) {}
// Disallow copy'ing
ReservedIndirectHeap(const ReservedIndirectHeap &) = delete;
ReservedIndirectHeap &operator=(const ReservedIndirectHeap &) = delete;
uint32_t getHeapSizeInPages() const override {
return parentHeapSizeInPages;
}
void setHeapSizeInPages(uint32_t value) {
parentHeapSizeInPages = value;
}
protected:
uint32_t parentHeapSizeInPages = 0;
};
} // namespace NEO

297
shared/test/unit_test/command_container/command_container_tests.cpp
View File

@ -937,13 +937,30 @@ TEST_F(CommandContainerTest, GivenCmdContainerWhenContainerIsInitializedThenSurf
HWTEST_F(CommandContainerTest, givenCmdContainerHasImmediateCsrWhenGettingHeapWithoutEnsuringSpaceThenExpectNullptrReturnedOrUnrecoverable) {
MyMockCommandContainer cmdContainer;
cmdContainer.enableHeapSharing();
cmdContainer.setImmediateCmdListCsr(pDevice->getDefaultEngine().commandStreamReceiver);
cmdContainer.immediateReusableAllocationList = std::make_unique<NEO::AllocationsList>();
auto &containerSshReserve = cmdContainer.getSurfaceStateHeapReserve();
EXPECT_NE(nullptr, containerSshReserve.indirectHeapReservation);
auto &containerDshReserve = cmdContainer.getDynamicStateHeapReserve();
EXPECT_NE(nullptr, containerDshReserve.indirectHeapReservation);
NEO::ReservedIndirectHeap reservedSsh(nullptr, false);
NEO::ReservedIndirectHeap reservedDsh(nullptr, false);
auto sshHeapPtr = &reservedSsh;
auto dshHeapPtr = &reservedDsh;
HeapReserveArguments sshReserveArgs = {sshHeapPtr, 0, 0};
HeapReserveArguments dshReserveArgs = {dshHeapPtr, 0, 0};
const size_t dshAlign = NEO::EncodeDispatchKernel<FamilyType>::getDefaultDshAlignment();
const size_t sshAlign = NEO::EncodeDispatchKernel<FamilyType>::getDefaultSshAlignment();
sshReserveArgs.alignment = sshAlign;
dshReserveArgs.alignment = dshAlign;
cmdContainer.enableHeapSharing();
cmdContainer.setImmediateCmdListCsr(pDevice->getDefaultEngine().commandStreamReceiver);
cmdContainer.immediateReusableAllocationList = std::make_unique<NEO::AllocationsList>();
cmdContainer.setNumIddPerBlock(1);
auto code = cmdContainer.initialize(pDevice, nullptr, true);
EXPECT_EQ(CommandContainer::ErrorCode::SUCCESS, code);
@ -960,11 +977,17 @@ HWTEST_F(CommandContainerTest, givenCmdContainerHasImmediateCsrWhenGettingHeapWi
auto &ultCsr = pDevice->getUltCommandStreamReceiver<FamilyType>();
ultCsr.recursiveLockCounter = 0;
cmdContainer.ensureHeapSizePrepared(0, sshAlign, 0, dshAlign, false);
sshReserveArgs.size = 0;
dshReserveArgs.size = 0;
cmdContainer.reserveSpaceForDispatch(sshReserveArgs, dshReserveArgs, false);
EXPECT_EQ(1u, ultCsr.recursiveLockCounter);
EXPECT_EQ(nullptr, cmdContainer.getIndirectHeap(HeapType::DYNAMIC_STATE));
EXPECT_EQ(nullptr, reservedDsh.getCpuBase());
EXPECT_NE(nullptr, cmdContainer.getIndirectHeap(HeapType::SURFACE_STATE));
EXPECT_NE(nullptr, reservedSsh.getCpuBase());
EXPECT_EQ(cmdContainer.getIndirectHeap(HeapType::SURFACE_STATE)->getCpuBase(), reservedSsh.getCpuBase());
EXPECT_THROW(cmdContainer.getHeapSpaceAllowGrow(HeapType::DYNAMIC_STATE, 0), std::exception);
EXPECT_THROW(cmdContainer.getHeapWithRequiredSizeAndAlignment(HeapType::DYNAMIC_STATE, 0, 0), std::exception);
@ -972,7 +995,7 @@ HWTEST_F(CommandContainerTest, givenCmdContainerHasImmediateCsrWhenGettingHeapWi
EXPECT_NO_THROW(cmdContainer.getHeapSpaceAllowGrow(HeapType::SURFACE_STATE, 0));
EXPECT_NO_THROW(cmdContainer.getHeapWithRequiredSizeAndAlignment(HeapType::SURFACE_STATE, 0, 0));
cmdContainer.ensureHeapSizePrepared(0, sshAlign, 0, dshAlign, true);
cmdContainer.reserveSpaceForDispatch(sshReserveArgs, dshReserveArgs, true);
EXPECT_EQ(2u, ultCsr.recursiveLockCounter);
ASSERT_NE(nullptr, cmdContainer.getIndirectHeap(HeapType::DYNAMIC_STATE));
@ -986,23 +1009,53 @@ HWTEST_F(CommandContainerTest, givenCmdContainerHasImmediateCsrWhenGettingHeapWi
size_t initSshSize = sizeUsedSsh;
constexpr size_t misAlignedSize = 3;
cmdContainer.ensureHeapSizePrepared(misAlignedSize, sshAlign, misAlignedSize, dshAlign, true);
sshReserveArgs.size = misAlignedSize;
dshReserveArgs.size = misAlignedSize;
cmdContainer.reserveSpaceForDispatch(sshReserveArgs, dshReserveArgs, true);
EXPECT_EQ(3u, ultCsr.recursiveLockCounter);
auto dshHeap = cmdContainer.getIndirectHeap(HeapType::DYNAMIC_STATE);
EXPECT_NE(nullptr, dshHeap);
sshHeap->getSpace(misAlignedSize);
dshHeap->getSpace(misAlignedSize);
EXPECT_EQ(sshHeapPtr->getCpuBase(), sshReserveArgs.indirectHeapReservation->getCpuBase());
EXPECT_EQ(dshHeapPtr->getCpuBase(), dshReserveArgs.indirectHeapReservation->getCpuBase());
cmdContainer.ensureHeapSizePrepared(sshAlign, sshAlign, dshAlign, dshAlign, true);
EXPECT_EQ(sshHeap->getCpuBase(), sshReserveArgs.indirectHeapReservation->getCpuBase());
EXPECT_EQ(dshHeap->getCpuBase(), dshReserveArgs.indirectHeapReservation->getCpuBase());
EXPECT_EQ(sshHeap->getHeapSizeInPages(), sshReserveArgs.indirectHeapReservation->getHeapSizeInPages());
EXPECT_EQ(dshHeap->getHeapSizeInPages(), dshReserveArgs.indirectHeapReservation->getHeapSizeInPages());
EXPECT_EQ(sshHeap->getGraphicsAllocation(), sshReserveArgs.indirectHeapReservation->getGraphicsAllocation());
EXPECT_EQ(dshHeap->getGraphicsAllocation(), dshReserveArgs.indirectHeapReservation->getGraphicsAllocation());
sshHeapPtr->getSpace(misAlignedSize);
dshHeapPtr->getSpace(misAlignedSize);
EXPECT_EQ(0u, sshHeapPtr->getAvailableSpace());
EXPECT_EQ(0u, dshHeapPtr->getAvailableSpace());
sshReserveArgs.size = sshAlign;
dshReserveArgs.size = dshAlign;
cmdContainer.reserveSpaceForDispatch(sshReserveArgs, dshReserveArgs, true);
EXPECT_EQ(4u, ultCsr.recursiveLockCounter);
sshHeap->align(sshAlign);
sshHeap->getSpace(sshAlign);
EXPECT_EQ(sshHeapPtr->getCpuBase(), sshReserveArgs.indirectHeapReservation->getCpuBase());
EXPECT_EQ(dshHeapPtr->getCpuBase(), dshReserveArgs.indirectHeapReservation->getCpuBase());
dshHeap->align(dshAlign);
dshHeap->getSpace(dshAlign);
sshHeapPtr->align(sshAlign);
sshHeapPtr->getSpace(sshAlign);
dshHeapPtr->align(dshAlign);
dshHeapPtr->getSpace(dshAlign);
EXPECT_EQ(0u, sshHeapPtr->getAvailableSpace());
EXPECT_EQ(0u, dshHeapPtr->getAvailableSpace());
EXPECT_EQ(sshHeap->getCpuBase(), sshReserveArgs.indirectHeapReservation->getCpuBase());
EXPECT_EQ(dshHeap->getCpuBase(), dshReserveArgs.indirectHeapReservation->getCpuBase());
EXPECT_EQ(sshHeap->getHeapSizeInPages(), sshReserveArgs.indirectHeapReservation->getHeapSizeInPages());
EXPECT_EQ(dshHeap->getHeapSizeInPages(), dshReserveArgs.indirectHeapReservation->getHeapSizeInPages());
EXPECT_EQ(sshHeap->getGraphicsAllocation(), sshReserveArgs.indirectHeapReservation->getGraphicsAllocation());
EXPECT_EQ(dshHeap->getGraphicsAllocation(), dshReserveArgs.indirectHeapReservation->getGraphicsAllocation());
sizeUsedDsh = dshHeap->getUsed();
sizeUsedSsh = sshHeap->getUsed();
@ -1010,7 +1063,11 @@ HWTEST_F(CommandContainerTest, givenCmdContainerHasImmediateCsrWhenGettingHeapWi
EXPECT_EQ(2 * sshAlign + initSshSize, sizeUsedSsh);
EXPECT_EQ(2 * dshAlign, sizeUsedDsh);
cmdContainer.ensureHeapSizePrepared(4 * MemoryConstants::kiloByte, sshAlign, 4 * MemoryConstants::kiloByte, dshAlign, true);
constexpr size_t nonZeroSshSize = 4 * MemoryConstants::kiloByte;
constexpr size_t nonZeroDshSize = 4 * MemoryConstants::kiloByte;
sshReserveArgs.size = nonZeroSshSize;
dshReserveArgs.size = nonZeroDshSize;
cmdContainer.reserveSpaceForDispatch(sshReserveArgs, dshReserveArgs, true);
EXPECT_EQ(5u, ultCsr.recursiveLockCounter);
dshHeap = cmdContainer.getIndirectHeap(HeapType::DYNAMIC_STATE);
@ -1018,14 +1075,33 @@ HWTEST_F(CommandContainerTest, givenCmdContainerHasImmediateCsrWhenGettingHeapWi
sshHeap = cmdContainer.getIndirectHeap(HeapType::SURFACE_STATE);
EXPECT_NE(nullptr, sshHeap);
EXPECT_EQ(sshHeap->getCpuBase(), sshReserveArgs.indirectHeapReservation->getCpuBase());
EXPECT_EQ(dshHeap->getCpuBase(), dshReserveArgs.indirectHeapReservation->getCpuBase());
EXPECT_EQ(sshHeap->getHeapSizeInPages(), sshReserveArgs.indirectHeapReservation->getHeapSizeInPages());
EXPECT_EQ(dshHeap->getHeapSizeInPages(), dshReserveArgs.indirectHeapReservation->getHeapSizeInPages());
EXPECT_EQ(sshHeap->getGraphicsAllocation(), sshReserveArgs.indirectHeapReservation->getGraphicsAllocation());
EXPECT_EQ(dshHeap->getGraphicsAllocation(), dshReserveArgs.indirectHeapReservation->getGraphicsAllocation());
sizeUsedDsh = dshHeap->getUsed();
sizeUsedSsh = sshHeap->getUsed();
void *dshPtr = cmdContainer.getHeapSpaceAllowGrow(HeapType::DYNAMIC_STATE, 64);
void *sshPtr = cmdContainer.getHeapSpaceAllowGrow(HeapType::SURFACE_STATE, 64);
size_t sizeReserveUsedDsh = reservedDsh.getUsed();
size_t sizeReserveUsedSsh = reservedSsh.getUsed();
EXPECT_EQ(ptrOffset(dshHeap->getCpuBase(), sizeUsedDsh), dshPtr);
EXPECT_EQ(ptrOffset(sshHeap->getCpuBase(), sizeUsedSsh), sshPtr);
EXPECT_EQ(sizeUsedDsh, sizeReserveUsedDsh + nonZeroDshSize);
EXPECT_EQ(sizeUsedSsh, sizeReserveUsedSsh + nonZeroSshSize);
EXPECT_EQ(nonZeroDshSize, reservedDsh.getAvailableSpace());
EXPECT_EQ(nonZeroSshSize, reservedSsh.getAvailableSpace());
EXPECT_EQ(sizeUsedDsh, reservedDsh.getMaxAvailableSpace());
EXPECT_EQ(sizeUsedSsh, reservedSsh.getMaxAvailableSpace());
void *dshReservePtr = reservedDsh.getSpace(64);
void *sshReservePtr = reservedSsh.getSpace(64);
EXPECT_EQ(ptrOffset(reservedDsh.getCpuBase(), sizeReserveUsedDsh), dshReservePtr);
EXPECT_EQ(ptrOffset(reservedSsh.getCpuBase(), sizeReserveUsedSsh), sshReservePtr);
auto alignedHeapDsh = cmdContainer.getHeapWithRequiredSizeAndAlignment(HeapType::DYNAMIC_STATE, 128, 128);
auto alignedHeapSsh = cmdContainer.getHeapWithRequiredSizeAndAlignment(HeapType::SURFACE_STATE, 128, 128);
@ -1052,24 +1128,203 @@ HWTEST_F(CommandContainerTest, givenCmdContainerUsedInRegularCmdListWhenGettingH
const size_t sshAlign = NEO::EncodeDispatchKernel<FamilyType>::getDefaultSshAlignment();
MyMockCommandContainer cmdContainer;
constexpr size_t zeroSize = 0;
NEO::ReservedIndirectHeap reservedSsh(nullptr, zeroSize);
NEO::ReservedIndirectHeap reservedDsh(nullptr, zeroSize);
auto sshHeapPtr = &reservedSsh;
auto dshHeapPtr = &reservedDsh;
HeapReserveArguments sshReserveArgs = {sshHeapPtr, 0, sshAlign};
HeapReserveArguments dshReserveArgs = {dshHeapPtr, 0, dshAlign};
auto code = cmdContainer.initialize(pDevice, nullptr, true);
EXPECT_EQ(CommandContainer::ErrorCode::SUCCESS, code);
cmdContainer.ensureHeapSizePrepared(0, sshAlign, 0, dshAlign, true);
cmdContainer.reserveSpaceForDispatch(sshReserveArgs, dshReserveArgs, true);
EXPECT_EQ(nullptr, sshReserveArgs.indirectHeapReservation);
EXPECT_EQ(nullptr, dshReserveArgs.indirectHeapReservation);
auto dsh = cmdContainer.getIndirectHeap(HeapType::DYNAMIC_STATE);
auto ssh = cmdContainer.getIndirectHeap(HeapType::SURFACE_STATE);
EXPECT_EQ(0u, reservedDsh.getAvailableSpace());
EXPECT_EQ(0u, reservedSsh.getAvailableSpace());
EXPECT_NE(nullptr, dsh);
EXPECT_NE(nullptr, ssh);
dsh->getSpace(dsh->getAvailableSpace() - 64);
cmdContainer.ensureHeapSizePrepared(4 * MemoryConstants::kiloByte, sshAlign, 4 * MemoryConstants::kiloByte, dshAlign, false);
constexpr size_t nonZeroSize = 4 * MemoryConstants::kiloByte;
sshReserveArgs.size = nonZeroSize;
dshReserveArgs.size = nonZeroSize;
sshReserveArgs.indirectHeapReservation = sshHeapPtr;
dshReserveArgs.indirectHeapReservation = dshHeapPtr;
cmdContainer.reserveSpaceForDispatch(sshReserveArgs, dshReserveArgs, true);
dsh = cmdContainer.getIndirectHeap(HeapType::DYNAMIC_STATE);
EXPECT_EQ(64u, dsh->getAvailableSpace());
EXPECT_EQ(dsh->getMaxAvailableSpace(), dsh->getAvailableSpace());
EXPECT_EQ(nullptr, sshReserveArgs.indirectHeapReservation);
EXPECT_EQ(nullptr, dshReserveArgs.indirectHeapReservation);
}
HWTEST_F(CommandContainerTest, givenCmdContainerUsingPrivateHeapsWhenGettingReserveHeapThenExpectReserveNullified) {
const bool dshSupport = pDevice->getDeviceInfo().imageSupport;
MyMockCommandContainer cmdContainer;
NEO::ReservedIndirectHeap reservedSsh(nullptr);
NEO::ReservedIndirectHeap reservedDsh(nullptr);
auto sshHeapPtr = &reservedSsh;
auto dshHeapPtr = &reservedDsh;
const size_t dshAlign = NEO::EncodeDispatchKernel<FamilyType>::getDefaultDshAlignment();
const size_t sshAlign = NEO::EncodeDispatchKernel<FamilyType>::getDefaultSshAlignment();
HeapReserveArguments sshReserveArgs = {sshHeapPtr, 0, sshAlign};
HeapReserveArguments dshReserveArgs = {dshHeapPtr, 0, dshAlign};
auto code = cmdContainer.initialize(pDevice, nullptr, true);
EXPECT_EQ(CommandContainer::ErrorCode::SUCCESS, code);
constexpr size_t nonZeroSshSize = 4 * MemoryConstants::kiloByte;
constexpr size_t nonZeroDshSize = 4 * MemoryConstants::kiloByte + 64;
sshReserveArgs.size = nonZeroSshSize;
dshReserveArgs.size = nonZeroDshSize;
cmdContainer.reserveSpaceForDispatch(sshReserveArgs, dshReserveArgs, dshSupport);
if (dshSupport) {
auto dshHeap = cmdContainer.getIndirectHeap(HeapType::DYNAMIC_STATE);
ASSERT_NE(nullptr, dshHeap);
EXPECT_EQ(nullptr, dshReserveArgs.indirectHeapReservation);
}
auto sshHeap = cmdContainer.getIndirectHeap(HeapType::SURFACE_STATE);
ASSERT_NE(nullptr, sshHeap);
EXPECT_EQ(nullptr, sshReserveArgs.indirectHeapReservation);
}
HWTEST_F(CommandContainerTest,
givenCmdContainerUsesSharedHeapsWhenGettingSpaceAfterMisalignedHeapCurrentPointerAndAlignmentIsProvidedThenExpectToProvideAlignmentPaddingToReserveHeap) {
const bool dshSupport = pDevice->getDeviceInfo().imageSupport;
MyMockCommandContainer cmdContainer;
NEO::ReservedIndirectHeap reservedSsh(nullptr, false);
NEO::ReservedIndirectHeap reservedDsh(nullptr, false);
auto sshHeapPtr = &reservedSsh;
auto dshHeapPtr = &reservedDsh;
constexpr size_t dshExampleAlignment = 64;
constexpr size_t sshExampleAlignment = 64;
HeapReserveArguments sshReserveArgs = {sshHeapPtr, 0, sshExampleAlignment};
HeapReserveArguments dshReserveArgs = {dshHeapPtr, 0, dshExampleAlignment};
cmdContainer.enableHeapSharing();
cmdContainer.setImmediateCmdListCsr(pDevice->getDefaultEngine().commandStreamReceiver);
cmdContainer.immediateReusableAllocationList = std::make_unique<NEO::AllocationsList>();
cmdContainer.setNumIddPerBlock(1);
auto code = cmdContainer.initialize(pDevice, nullptr, true);
EXPECT_EQ(CommandContainer::ErrorCode::SUCCESS, code);
constexpr size_t misalignedSize = 11;
sshReserveArgs.size = misalignedSize;
dshReserveArgs.size = misalignedSize;
cmdContainer.reserveSpaceForDispatch(sshReserveArgs, dshReserveArgs, dshSupport);
size_t oldUsedDsh = 0;
if (dshSupport) {
auto dshHeap = cmdContainer.getIndirectHeap(HeapType::DYNAMIC_STATE);
ASSERT_NE(nullptr, dshHeap);
size_t sizeUsedDsh = dshHeap->getUsed();
size_t sizeReserveUsedDsh = reservedDsh.getUsed();
EXPECT_EQ(sizeUsedDsh, sizeReserveUsedDsh + misalignedSize);
EXPECT_EQ(misalignedSize, reservedDsh.getAvailableSpace());
EXPECT_EQ(sizeUsedDsh, reservedDsh.getMaxAvailableSpace());
void *dshReservePtr = reservedDsh.getSpace(8);
EXPECT_EQ(ptrOffset(reservedDsh.getCpuBase(), sizeReserveUsedDsh), dshReservePtr);
oldUsedDsh = sizeUsedDsh;
}
size_t oldUsedSsh = 0;
auto sshHeap = cmdContainer.getIndirectHeap(HeapType::SURFACE_STATE);
ASSERT_NE(nullptr, sshHeap);
size_t sizeUsedSsh = sshHeap->getUsed();
size_t sizeReserveUsedSsh = reservedSsh.getUsed();
EXPECT_EQ(sizeUsedSsh, sizeReserveUsedSsh + misalignedSize);
EXPECT_EQ(misalignedSize, reservedSsh.getAvailableSpace());
EXPECT_EQ(sizeUsedSsh, reservedSsh.getMaxAvailableSpace());
void *sshReservePtr = reservedSsh.getSpace(8);
EXPECT_EQ(ptrOffset(reservedSsh.getCpuBase(), sizeReserveUsedSsh), sshReservePtr);
oldUsedSsh = sizeUsedSsh;
constexpr size_t zeroSize = 0;
sshReserveArgs.size = zeroSize;
dshReserveArgs.size = zeroSize;
cmdContainer.reserveSpaceForDispatch(sshReserveArgs, dshReserveArgs, dshSupport);
if (dshSupport) {
auto dshHeap = cmdContainer.getIndirectHeap(HeapType::DYNAMIC_STATE);
ASSERT_NE(nullptr, dshHeap);
size_t sizeUsedDsh = dshHeap->getUsed();
size_t sizeReserveUsedDsh = reservedDsh.getUsed();
EXPECT_EQ(oldUsedDsh, sizeUsedDsh);
EXPECT_EQ(zeroSize, reservedDsh.getAvailableSpace());
EXPECT_EQ(sizeReserveUsedDsh, reservedDsh.getMaxAvailableSpace());
}
sshHeap = cmdContainer.getIndirectHeap(HeapType::SURFACE_STATE);
ASSERT_NE(nullptr, sshHeap);
sizeUsedSsh = sshHeap->getUsed();
sizeReserveUsedSsh = reservedSsh.getUsed();
EXPECT_EQ(oldUsedSsh, sizeUsedSsh);
EXPECT_EQ(zeroSize, reservedSsh.getAvailableSpace());
EXPECT_EQ(sizeReserveUsedSsh, reservedSsh.getMaxAvailableSpace());
sshReserveArgs.size = misalignedSize;
dshReserveArgs.size = misalignedSize;
cmdContainer.reserveSpaceForDispatch(sshReserveArgs, dshReserveArgs, dshSupport);
if (dshSupport) {
auto dshHeap = cmdContainer.getIndirectHeap(HeapType::DYNAMIC_STATE);
ASSERT_NE(nullptr, dshHeap);
size_t alignedDshSize = alignUp(misalignedSize, dshExampleAlignment);
size_t sizeUsedDsh = dshHeap->getUsed();
size_t sizeReserveUsedDsh = reservedDsh.getUsed();
EXPECT_EQ(sizeUsedDsh, sizeReserveUsedDsh + alignedDshSize);
EXPECT_EQ(alignedDshSize, reservedDsh.getAvailableSpace());
EXPECT_EQ(sizeUsedDsh, reservedDsh.getMaxAvailableSpace());
void *dshReservePtr = reservedDsh.getSpace(4);
EXPECT_EQ(ptrOffset(reservedDsh.getCpuBase(), sizeReserveUsedDsh), dshReservePtr);
}
sshHeap = cmdContainer.getIndirectHeap(HeapType::SURFACE_STATE);
ASSERT_NE(nullptr, sshHeap);
size_t alignedSshSize = alignUp(misalignedSize, sshExampleAlignment);
sizeUsedSsh = sshHeap->getUsed();
sizeReserveUsedSsh = reservedSsh.getUsed();
EXPECT_EQ(sizeUsedSsh, sizeReserveUsedSsh + alignedSshSize);
EXPECT_EQ(alignedSshSize, reservedSsh.getAvailableSpace());
EXPECT_EQ(sizeUsedSsh, reservedSsh.getMaxAvailableSpace());
sshReservePtr = reservedSsh.getSpace(4);
EXPECT_EQ(ptrOffset(reservedSsh.getCpuBase(), sizeReserveUsedSsh), sshReservePtr);
}
struct MockHeapHelper : public HeapHelper {