Change fences to use tag allocation

Instead of creating new allocation per fence, use the task count.
Fence synchronize will wait for task count update.

Related-To: NEO-6634

Signed-off-by: Dominik Dabek <dominik.dabek@intel.com>
This commit is contained in:
Dominik Dabek
2022-01-31 11:06:18 +00:00
committed by Compute-Runtime-Automation
parent 8a4d28ef8b
commit fb1a008414
6 changed files with 84 additions and 154 deletions

View File

@@ -115,7 +115,6 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandLists(
size_t preemptionSize = 0u;
size_t debuggerCmdsSize = 0;
constexpr size_t residencyContainerSpaceForPreemption = 2;
constexpr size_t residencyContainerSpaceForFence = 1;
constexpr size_t residencyContainerSpaceForTagWrite = 1;
NEO::Device *neoDevice = device->getNEODevice();
@@ -201,7 +200,6 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandLists(
size_t linearStreamSizeEstimate = totalCmdBuffers * sizeof(MI_BATCH_BUFFER_START);
linearStreamSizeEstimate += csr->getCmdsSizeForHardwareContext();
if (directSubmissionEnabled) {
linearStreamSizeEstimate += sizeof(MI_BATCH_BUFFER_START);
} else {
@@ -212,14 +210,10 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandLists(
if (programActivePartitionConfig) {
linearStreamSizeEstimate += csrHw->getCmdSizeForActivePartitionConfig();
}
const auto &hwInfo = this->device->getHwInfo();
if (hFence) {
fence = Fence::fromHandle(hFence);
spaceForResidency += residencyContainerSpaceForFence;
linearStreamSizeEstimate += isCopyOnlyCommandQueue ? NEO::EncodeMiFlushDW<GfxFamily>::getMiFlushDwCmdSizeForDataWrite() : NEO::MemorySynchronizationCommands<GfxFamily>::getSizeForPipeControlWithPostSyncOperation(hwInfo);
}
spaceForResidency += residencyContainerSpaceForTagWrite;
csr->getResidencyAllocations().reserve(spaceForResidency);
@@ -420,11 +414,12 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandLists(
commandQueuePreemptionMode = statePreemption;
if (hFence) {
csr->makeResident(fence->getAllocation());
fence = Fence::fromHandle(hFence);
fence->assignTaskCountFromCsr();
if (isCopyOnlyCommandQueue) {
NEO::MiFlushArgs args;
args.commandWithPostSync = true;
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(child, fence->getGpuAddress(), Fence::STATE_SIGNALED, args, hwInfo);
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(child, csr->getTagAllocation()->getGpuAddress(), csr->peekTaskCount() + 1, args, hwInfo);
} else {
NEO::PipeControlArgs args;
args.dcFlushEnable = NEO::MemorySynchronizationCommands<GfxFamily>::getDcFlushEnable(true, hwInfo);
@@ -434,8 +429,8 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandLists(
fence->setPartitionCount(partitionCount);
NEO::MemorySynchronizationCommands<GfxFamily>::addPipeControlAndProgramPostSyncOperation(
child, POST_SYNC_OPERATION::POST_SYNC_OPERATION_WRITE_IMMEDIATE_DATA,
fence->getGpuAddress(),
Fence::STATE_SIGNALED,
csr->getTagAllocation()->getGpuAddress(),
csr->peekTaskCount() + 1,
hwInfo,
args);
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -18,52 +18,27 @@ namespace L0 {
Fence *Fence::create(CommandQueueImp *cmdQueue, const ze_fence_desc_t *desc) {
auto fence = new FenceImp(cmdQueue);
UNRECOVERABLE_IF(fence == nullptr);
fence->initialize();
fence->reset();
return fence;
}
FenceImp::~FenceImp() {
cmdQueue->getDevice()->getDriverHandle()->getMemoryManager()->freeGraphicsMemory(allocation);
allocation = nullptr;
}
ze_result_t FenceImp::queryStatus() {
auto csr = cmdQueue->getCsr();
csr->downloadAllocations();
volatile uint32_t *hostAddr = static_cast<uint32_t *>(allocation->getUnderlyingBuffer());
uint32_t queryVal = Fence::STATE_CLEARED;
for (uint32_t i = 0; i < partitionCount; i++) {
queryVal = *hostAddr;
if (queryVal == Fence::STATE_CLEARED) {
break;
}
hostAddr = ptrOffset(hostAddr, csr->getPostSyncWriteOffset());
}
return queryVal == Fence::STATE_CLEARED ? ZE_RESULT_NOT_READY : ZE_RESULT_SUCCESS;
auto *hostAddr = csr->getTagAddress();
return csr->testTaskCountReady(hostAddr, taskCount) ? ZE_RESULT_SUCCESS : ZE_RESULT_NOT_READY;
}
void FenceImp::initialize() {
NEO::AllocationProperties properties(
cmdQueue->getDevice()->getRootDeviceIndex(), MemoryConstants::pageSize, NEO::GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY, cmdQueue->getDevice()->getNEODevice()->getDeviceBitfield());
properties.alignment = MemoryConstants::pageSize;
allocation = cmdQueue->getDevice()->getDriverHandle()->getMemoryManager()->allocateGraphicsMemoryWithProperties(properties);
UNRECOVERABLE_IF(allocation == nullptr);
reset();
ze_result_t FenceImp::assignTaskCountFromCsr() {
auto csr = cmdQueue->getCsr();
taskCount = csr->peekTaskCount() + 1;
return ZE_RESULT_SUCCESS;
}
ze_result_t FenceImp::reset() {
constexpr uint32_t maxPartitionCount = 16;
volatile uint32_t *hostAddress = static_cast<uint32_t *>(allocation->getUnderlyingBuffer());
for (uint32_t i = 0; i < maxPartitionCount; i++) {
*hostAddress = Fence::STATE_CLEARED;
NEO::CpuIntrinsics::clFlush(const_cast<uint32_t *>(hostAddress));
hostAddress = ptrOffset(hostAddress, cmdQueue->getCsr()->getPostSyncWriteOffset());
}
partitionCount = 1;
taskCount = std::numeric_limits<uint32_t>::max();
return ZE_RESULT_SUCCESS;
}
@@ -76,6 +51,10 @@ ze_result_t FenceImp::hostSynchronize(uint64_t timeout) {
return ZE_RESULT_SUCCESS;
}
if (std::numeric_limits<uint32_t>::max() == taskCount) {
return ZE_RESULT_NOT_READY;
}
if (timeout == 0) {
return queryStatus();
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Intel Corporation
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -25,42 +25,25 @@ struct Fence : _ze_fence_handle_t {
virtual ze_result_t destroy() = 0;
virtual ze_result_t hostSynchronize(uint64_t timeout) = 0;
virtual ze_result_t queryStatus() = 0;
virtual ze_result_t assignTaskCountFromCsr() = 0;
virtual ze_result_t reset() = 0;
static Fence *fromHandle(ze_fence_handle_t handle) { return static_cast<Fence *>(handle); }
inline ze_fence_handle_t toHandle() { return this; }
enum State : uint32_t {
STATE_SIGNALED = 0u,
STATE_CLEARED = std::numeric_limits<uint32_t>::max(),
STATE_INITIAL = STATE_CLEARED
};
enum EnqueueState : uint32_t { ENQUEUE_NOT_READY = 0u,
ENQUEUE_READY };
NEO::GraphicsAllocation &getAllocation() const { return *allocation; }
uint64_t getGpuAddress() {
UNRECOVERABLE_IF(allocation == nullptr);
return allocation->getGpuAddress();
}
void setPartitionCount(uint32_t newPartitionCount) {
partitionCount = newPartitionCount;
}
protected:
NEO::GraphicsAllocation *allocation = nullptr;
uint32_t partitionCount = 1;
uint32_t taskCount = 0;
};
struct FenceImp : public Fence {
FenceImp(CommandQueueImp *cmdQueueImp) : cmdQueue(cmdQueueImp) {}
~FenceImp() override;
ze_result_t destroy() override {
delete this;
return ZE_RESULT_SUCCESS;
@@ -70,9 +53,9 @@ struct FenceImp : public Fence {
ze_result_t queryStatus() override;
ze_result_t reset() override;
ze_result_t assignTaskCountFromCsr() override;
void initialize();
ze_result_t reset() override;
protected:
CommandQueueImp *cmdQueue;

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2021 Intel Corporation
* Copyright (C) 2021-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -19,29 +19,21 @@ namespace ult {
template <>
struct WhiteBox<::L0::Fence> : public ::L0::Fence {
using ::L0::Fence::allocation;
using ::L0::Fence::partitionCount;
using ::L0::Fence::taskCount;
};
using Fence = WhiteBox<::L0::Fence>;
template <>
struct Mock<Fence> : public Fence {
Mock() : mockAllocation(0, NEO::GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY,
&memory, reinterpret_cast<uint64_t>(&memory), 0, sizeof(memory),
MemoryPool::System4KBPages) {
allocation = &mockAllocation;
}
~Mock() override = default;
ADDMETHOD_NOBASE(destroy, ze_result_t, ZE_RESULT_SUCCESS, ());
ADDMETHOD_NOBASE(hostSynchronize, ze_result_t, ZE_RESULT_SUCCESS, (uint64_t timeout));
ADDMETHOD_NOBASE(queryStatus, ze_result_t, ZE_RESULT_SUCCESS, ());
ADDMETHOD_NOBASE(assignTaskCountFromCsr, ze_result_t, ZE_RESULT_SUCCESS, ());
ADDMETHOD_NOBASE(reset, ze_result_t, ZE_RESULT_SUCCESS, ());
// Fake an allocation for event memory
alignas(16) uint32_t memory = -1;
NEO::GraphicsAllocation mockAllocation;
};
} // namespace ult

View File

@@ -11,6 +11,8 @@
#include "shared/source/utilities/software_tags_manager.h"
#include "shared/test/common/cmd_parse/gen_cmd_parse.h"
#include "shared/test/common/helpers/unit_test_helper.h"
#include "shared/test/common/libult/ult_command_stream_receiver.h"
#include "shared/test/common/mocks/mock_device.h"
#include "shared/test/common/mocks/ult_device_factory.h"
#include "shared/test/common/test_macros/test.h"
@@ -170,7 +172,41 @@ HWTEST_F(CommandQueueExecuteCommandLists, whenASecondLevelBatchBufferPerCommandL
commandQueue->destroy();
}
HWTEST2_F(CommandQueueExecuteCommandLists, whenUsingFenceThenExpectEndingPipeControlUpdatingFenceAllocation, IsGen9) {
HWTEST_F(CommandQueueExecuteCommandLists, givenFenceWhenExecutingCmdListThenFenceStatusIsCorrect) {
const ze_command_queue_desc_t desc{};
ze_result_t returnValue;
auto commandQueue = whitebox_cast(CommandQueue::create(productFamily,
device,
neoDevice->getDefaultEngine().commandStreamReceiver,
&desc,
false,
false,
returnValue));
ASSERT_NE(nullptr, commandQueue->commandStream);
auto &csr = neoDevice->getUltCommandStreamReceiver<FamilyType>();
*csr.tagAddress = 10;
csr.taskCount = 10;
ze_fence_desc_t fenceDesc{};
auto fence = whitebox_cast(Fence::create(commandQueue, &fenceDesc));
ASSERT_NE(nullptr, fence);
EXPECT_EQ(ZE_RESULT_NOT_READY, fence->queryStatus());
auto result = commandQueue->executeCommandLists(numCommandLists, commandLists, fence, true);
*csr.tagAddress = 11;
ASSERT_EQ(ZE_RESULT_SUCCESS, result);
EXPECT_EQ(*csr.tagAddress, fence->taskCount);
EXPECT_EQ(ZE_RESULT_SUCCESS, fence->queryStatus());
//reset fence
fence->assignTaskCountFromCsr();
EXPECT_EQ(ZE_RESULT_NOT_READY, fence->queryStatus());
fence->destroy();
commandQueue->destroy();
}
HWTEST2_F(CommandQueueExecuteCommandLists, whenUsingFenceThenExpectEndingPipeControlUpdatingTagAllocation, IsGen9) {
using PIPE_CONTROL = typename FamilyType::PIPE_CONTROL;
using POST_SYNC_OPERATION = typename FamilyType::PIPE_CONTROL::POST_SYNC_OPERATION;
using PARSE = typename FamilyType::PARSE;
@@ -213,12 +249,11 @@ HWTEST2_F(CommandQueueExecuteCommandLists, whenUsingFenceThenExpectEndingPipeCon
ASSERT_LE(1u, pipeControls.size());
PIPE_CONTROL *fenceUpdate = genCmdCast<PIPE_CONTROL *>(*pipeControls[pipeControls.size() - 3]);
EXPECT_EQ(fence->getGpuAddress(), NEO::UnitTestHelper<FamilyType>::getPipeControlPostSyncAddress(*fenceUpdate));
EXPECT_EQ(commandQueue->getCsr()->getTagAllocation()->getGpuAddress(), NEO::UnitTestHelper<FamilyType>::getPipeControlPostSyncAddress(*fenceUpdate));
EXPECT_EQ(POST_SYNC_OPERATION::POST_SYNC_OPERATION_WRITE_IMMEDIATE_DATA, fenceUpdate->getPostSyncOperation());
uint64_t fenceImmData = Fence::STATE_SIGNALED;
EXPECT_EQ(fenceImmData, fenceUpdate->getImmediateData());
EXPECT_EQ(fence->taskCount, fenceUpdate->getImmediateData());
fence->destroy();
commandQueue->destroy();

View File

@@ -21,16 +21,12 @@ namespace ult {
using FenceTest = Test<DeviceFixture>;
TEST_F(FenceTest, whenQueryingStatusThenCsrAllocationsAreDownloaded) {
auto csr = std::make_unique<MockCommandStreamReceiver>(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield());
*csr->tagAddress = 0;
Mock<CommandQueue> cmdQueue(device, csr.get());
auto fence = Fence::create(&cmdQueue, nullptr);
EXPECT_NE(nullptr, fence);
auto &graphicsAllocation = fence->getAllocation();
EXPECT_EQ(neoDevice->getRootDeviceIndex(), graphicsAllocation.getRootDeviceIndex());
EXPECT_FALSE(csr->downloadAllocationsCalled);
auto status = fence->queryStatus();
@@ -43,6 +39,7 @@ TEST_F(FenceTest, whenQueryingStatusThenCsrAllocationsAreDownloaded) {
TEST_F(FenceTest, whenQueryingStatusWithoutCsrAndFenceUnsignaledThenReturnsNotReady) {
auto csr = std::make_unique<MockCommandStreamReceiver>(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield());
*csr->tagAddress = 0;
Mock<CommandQueue> cmdQueue(device, csr.get());
auto fence = Fence::create(&cmdQueue, nullptr);
@@ -52,23 +49,9 @@ TEST_F(FenceTest, whenQueryingStatusWithoutCsrAndFenceUnsignaledThenReturnsNotRe
fence->destroy();
}
TEST_F(FenceTest, whenQueryingStatusAndStateSignaledThenReturnSuccess) {
auto csr = std::make_unique<MockCommandStreamReceiver>(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield());
Mock<CommandQueue> cmdQueue(device, csr.get());
auto fence = Fence::create(&cmdQueue, nullptr);
EXPECT_NE(nullptr, fence);
auto &graphicsAllocation = fence->getAllocation();
auto hostAddr = static_cast<uint32_t *>(graphicsAllocation.getUnderlyingBuffer());
*hostAddr = Fence::STATE_SIGNALED;
auto status = fence->queryStatus();
EXPECT_EQ(ZE_RESULT_SUCCESS, status);
fence->destroy();
}
using FenceSynchronizeTest = Test<DeviceFixture>;
TEST_F(FenceSynchronizeTest, givenCallToFenceHostSynchronizeWithTimeoutZeroAndStateInitialHostSynchronizeReturnsNotReady) {
TEST_F(FenceSynchronizeTest, givenCallToFenceHostSynchronizeWithTimeoutZeroAndStateInitialThenHostSynchronizeReturnsNotReady) {
std::unique_ptr<MockCommandStreamReceiver> csr = nullptr;
csr = std::make_unique<MockCommandStreamReceiver>(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield());
@@ -76,12 +59,12 @@ TEST_F(FenceSynchronizeTest, givenCallToFenceHostSynchronizeWithTimeoutZeroAndSt
std::unique_ptr<L0::Fence> fence;
fence = std::unique_ptr<L0::Fence>(L0::Fence::create(&cmdQueue, nullptr));
EXPECT_NE(nullptr, fence);
*csr->tagAddress = 0;
ze_result_t result = fence->hostSynchronize(0);
EXPECT_EQ(ZE_RESULT_NOT_READY, result);
}
TEST_F(FenceSynchronizeTest, givenCallToFenceHostSynchronizeWithNonZeroTimeoutAndStateInitialHostSynchronizeReturnsNotReady) {
TEST_F(FenceSynchronizeTest, givenCallToFenceHostSynchronizeWithNonZeroTimeoutAndStateInitialThenHostSynchronizeReturnsNotReady) {
std::unique_ptr<MockCommandStreamReceiver> csr = nullptr;
csr = std::make_unique<MockCommandStreamReceiver>(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield());
@@ -89,75 +72,39 @@ TEST_F(FenceSynchronizeTest, givenCallToFenceHostSynchronizeWithNonZeroTimeoutAn
std::unique_ptr<L0::Fence> fence;
fence = std::unique_ptr<L0::Fence>(L0::Fence::create(&cmdQueue, nullptr));
EXPECT_NE(nullptr, fence);
*csr->tagAddress = 0;
ze_result_t result = fence->hostSynchronize(10);
EXPECT_EQ(ZE_RESULT_NOT_READY, result);
}
TEST_F(FenceSynchronizeTest, givenCallToFenceHostSynchronizeWithTimeoutZeroAndStateSignaledHostSynchronizeReturnsSuccess) {
TEST_F(FenceSynchronizeTest, givenCallToFenceHostSynchronizeWithTimeoutZeroAndTaskCountEqualsTagAllocationThenHostSynchronizeReturnsSuccess) {
std::unique_ptr<MockCommandStreamReceiver> csr = nullptr;
csr = std::make_unique<MockCommandStreamReceiver>(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield());
Mock<CommandQueue> cmdQueue(device, csr.get());
std::unique_ptr<L0::Fence> fence;
fence = std::unique_ptr<L0::Fence>(L0::Fence::create(&cmdQueue, nullptr));
auto fence = std::unique_ptr<Fence>(whitebox_cast(Fence::create(&cmdQueue, nullptr)));
EXPECT_NE(nullptr, fence);
auto alloc = &(fence->getAllocation());
auto hostAddr = static_cast<uint32_t *>(alloc->getUnderlyingBuffer());
*hostAddr = Fence::STATE_SIGNALED;
fence->taskCount = 1;
*csr->tagAddress = 1;
ze_result_t result = fence->hostSynchronize(0);
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
}
TEST_F(FenceSynchronizeTest, givenCallToFenceHostSynchronizeWithTimeoutNonZeroAndStateSignaledHostSynchronizeReturnsSuccess) {
TEST_F(FenceSynchronizeTest, givenCallToFenceHostSynchronizeWithTimeoutNonZeroAndTaskCountEqualsTagAllocationThenHostSynchronizeReturnsSuccess) {
std::unique_ptr<MockCommandStreamReceiver> csr = nullptr;
csr = std::make_unique<MockCommandStreamReceiver>(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield());
Mock<CommandQueue> cmdQueue(device, csr.get());
std::unique_ptr<L0::Fence> fence;
fence = std::unique_ptr<L0::Fence>(L0::Fence::create(&cmdQueue, nullptr));
auto fence = std::unique_ptr<Fence>(whitebox_cast(Fence::create(&cmdQueue, nullptr)));
EXPECT_NE(nullptr, fence);
auto alloc = &(fence->getAllocation());
auto hostAddr = static_cast<uint32_t *>(alloc->getUnderlyingBuffer());
*hostAddr = Fence::STATE_SIGNALED;
fence->taskCount = 1;
*csr->tagAddress = 1;
ze_result_t result = fence->hostSynchronize(10);
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
}
TEST_F(FenceSynchronizeTest, givenMultiplePartitionsWhenFenceIsResetThenAllPartitionFenceStatesAreReset) {
std::unique_ptr<MockCommandStreamReceiver> csr = nullptr;
csr = std::make_unique<MockCommandStreamReceiver>(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield());
csr->postSyncWriteOffset = 16;
Mock<CommandQueue> cmdQueue(device, csr.get());
auto fence = whitebox_cast(Fence::create(&cmdQueue, nullptr));
EXPECT_NE(nullptr, fence);
auto alloc = &(fence->getAllocation());
auto hostAddr = static_cast<uint32_t *>(alloc->getUnderlyingBuffer());
for (uint32_t i = 0; i < 16; i++) {
EXPECT_EQ(Fence::STATE_CLEARED, *hostAddr);
hostAddr = ptrOffset(hostAddr, 16);
}
hostAddr = static_cast<uint32_t *>(alloc->getUnderlyingBuffer());
fence->partitionCount = 2;
*hostAddr = Fence::STATE_SIGNALED;
hostAddr = ptrOffset(hostAddr, 16);
*hostAddr = Fence::STATE_SIGNALED;
ze_result_t result = fence->reset();
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
hostAddr = static_cast<uint32_t *>(alloc->getUnderlyingBuffer());
for (uint32_t i = 0; i < 16; i++) {
EXPECT_EQ(Fence::STATE_CLEARED, *hostAddr);
hostAddr = ptrOffset(hostAddr, 16);
}
EXPECT_EQ(1u, fence->partitionCount);
fence->destroy();
}
using FenceAubCsrTest = Test<DeviceFixture>;
HWTEST_F(FenceAubCsrTest, givenCallToFenceHostSynchronizeWithAubModeCsrReturnsSuccess) {
@@ -173,16 +120,15 @@ HWTEST_F(FenceAubCsrTest, givenCallToFenceHostSynchronizeWithAubModeCsrReturnsSu
driverHandle = std::make_unique<Mock<L0::DriverHandleImp>>();
driverHandle->initialize(std::move(devices));
device = driverHandle->devices[0];
int32_t tag;
int32_t tag = 1;
auto aubCsr = new MockCsrAub<FamilyType>(tag, *neoDevice->executionEnvironment, neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield());
neoDevice->resetCommandStreamReceiver(aubCsr);
Mock<CommandQueue> cmdQueue(device, aubCsr);
auto fence = Fence::create(&cmdQueue, nullptr);
auto fence = std::unique_ptr<L0::Fence>(Fence::create(&cmdQueue, nullptr));
ze_result_t result = fence->hostSynchronize(10);
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
fence->destroy();
}
} // namespace ult