intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2026-01-08 14:02:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add new atomic operation

Browse Source 
Related-To: NEO-5244

Signed-off-by: Zbigniew Zdanowicz <zbigniew.zdanowicz@intel.com>
This commit is contained in:
Zbigniew Zdanowicz
2021-03-04 17:54:14 +00:00
committed by Compute-Runtime-Automation
parent 3a91bcfb9b
commit 8f91fcdd73
13 changed files with 249 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
opencl/source/gen8/device_queue_gen8.cpp
View File

@@ -38,7 +38,7 @@ void DeviceQueueHw<Family>::addMiAtomicCmdWa(uint64_t atomicOpPlaceholder) {
atomicOpPlaceholder,
Family::MI_ATOMIC::ATOMIC_OPCODES::ATOMIC_8B_INCREMENT,
Family::MI_ATOMIC::DATA_SIZE::DATA_SIZE_QWORD,
0x1u, 0x1u);
0x1u, 0x1u, 0x0u, 0x0u);
}
template <>

2
opencl/source/gen9/device_queue_gen9.cpp
View File

@@ -44,7 +44,7 @@ void DeviceQueueHw<Family>::addMiAtomicCmdWa(uint64_t atomicOpPlaceholder) {
atomicOpPlaceholder,
Family::MI_ATOMIC::ATOMIC_OPCODES::ATOMIC_8B_INCREMENT,
Family::MI_ATOMIC::DATA_SIZE::DATA_SIZE_QWORD,
0x1u, 0x1u);
0x1u, 0x1u, 0x0u, 0x0u);
}
template <>

1
opencl/test/unit_test/aub_tests/command_stream/CMakeLists.txt
View File

@@ -11,6 +11,7 @@ target_sources(igdrcl_aub_tests PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/aub_command_stream_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/aub_mem_dump_tests.h
${CMAKE_CURRENT_SOURCE_DIR}/aub_mem_dump_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/aub_mi_atomic_tests.cpp
)
add_subdirectories()

147
opencl/test/unit_test/aub_tests/command_stream/aub_mi_atomic_tests.cpp Normal file
View File

@@ -0,0 +1,147 @@
/*
* Copyright (C) 2021 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/command_stream/command_stream_receiver_hw.h"
#include "shared/test/common/helpers/dispatch_flags_helper.h"
#include "opencl/source/command_stream/aub_command_stream_receiver_hw.h"
#include "opencl/test/unit_test/aub_tests/fixtures/aub_fixture.h"
#include "test.h"
#include <cstdint>
using namespace NEO;
struct MiAtomicAubFixture : public AUBFixture {
void SetUp() {
AUBFixture::SetUp(nullptr);
auto memoryManager = this->device->getMemoryManager();
AllocationProperties commandBufferProperties = {device->getRootDeviceIndex(),
true,
MemoryConstants::pageSize,
GraphicsAllocation::AllocationType::COMMAND_BUFFER,
false,
device->getDeviceBitfield()};
streamAllocation = memoryManager->allocateGraphicsMemoryWithProperties(commandBufferProperties);
ASSERT_NE(nullptr, streamAllocation);
AllocationProperties deviceBufferProperties = {device->getRootDeviceIndex(),
true,
MemoryConstants::pageSize,
GraphicsAllocation::AllocationType::BUFFER,
false,
device->getDeviceBitfield()};
deviceSurface = memoryManager->allocateGraphicsMemoryWithProperties(deviceBufferProperties);
ASSERT_NE(nullptr, deviceSurface);
AllocationProperties systemBufferProperties = {device->getRootDeviceIndex(),
true,
MemoryConstants::pageSize,
GraphicsAllocation::AllocationType::SVM_CPU,
false,
device->getDeviceBitfield()};
systemSurface = memoryManager->allocateGraphicsMemoryWithProperties(systemBufferProperties);
ASSERT_NE(nullptr, systemSurface);
taskStream.replaceGraphicsAllocation(streamAllocation);
taskStream.replaceBuffer(streamAllocation->getUnderlyingBuffer(),
streamAllocation->getUnderlyingBufferSize());
}
void TearDown() {
auto memoryManager = this->device->getMemoryManager();
memoryManager->freeGraphicsMemory(streamAllocation);
memoryManager->freeGraphicsMemory(deviceSurface);
memoryManager->freeGraphicsMemory(systemSurface);
AUBFixture::TearDown();
}
void flushStream() {
DispatchFlags dispatchFlags = DispatchFlagsHelper::createDefaultDispatchFlags();
dispatchFlags.guardCommandBufferWithPipeControl = true;
csr->makeResident(*deviceSurface);
csr->makeResident(*systemSurface);
csr->flushTask(taskStream, 0,
csr->getIndirectHeap(IndirectHeap::DYNAMIC_STATE, 0u),
csr->getIndirectHeap(IndirectHeap::INDIRECT_OBJECT, 0u),
csr->getIndirectHeap(IndirectHeap::SURFACE_STATE, 0u),
0u, dispatchFlags, device->getDevice());
csr->flushBatchedSubmissions();
}
LinearStream taskStream;
GraphicsAllocation *streamAllocation = nullptr;
GraphicsAllocation *deviceSurface = nullptr;
GraphicsAllocation *systemSurface = nullptr;
};
using MiAtomicAubTest = Test<MiAtomicAubFixture>;
HWTEST_F(MiAtomicAubTest, WhenDispatchingAtomicMoveOperationThenExpectCorrectEndValues) {
using MI_ATOMIC = typename FamilyType::MI_ATOMIC;
auto atomicAddress = deviceSurface->getGpuAddress();
auto expectedGpuAddressDwordOp1 = atomicAddress;
auto expectedGpuAddressDwordOp2 = expectedGpuAddressDwordOp1 + sizeof(uint32_t);
auto expectedGpuAddressQwordOp3 = expectedGpuAddressDwordOp2 + sizeof(uint32_t);
uint32_t operation1dword0 = 0x10;
EncodeAtomic<FamilyType>::programMiAtomic(taskStream, expectedGpuAddressDwordOp1,
MI_ATOMIC::ATOMIC_OPCODES::ATOMIC_4B_MOVE,
MI_ATOMIC::DATA_SIZE::DATA_SIZE_DWORD,
0, 0, operation1dword0, 0u);
uint32_t operation2dword0 = 0x22;
EncodeAtomic<FamilyType>::programMiAtomic(taskStream, expectedGpuAddressDwordOp2,
MI_ATOMIC::ATOMIC_OPCODES::ATOMIC_4B_MOVE,
MI_ATOMIC::DATA_SIZE::DATA_SIZE_DWORD,
0, 0, operation2dword0, 0u);
uint32_t operation3dword0 = 0xF0;
uint32_t operation3dword1 = 0x1F;
EncodeAtomic<FamilyType>::programMiAtomic(taskStream, expectedGpuAddressQwordOp3,
MI_ATOMIC::ATOMIC_OPCODES::ATOMIC_8B_MOVE,
MI_ATOMIC::DATA_SIZE::DATA_SIZE_QWORD,
0, 0, operation3dword0, operation3dword1);
uint64_t operation3qword = (static_cast<uint64_t>(operation3dword1) << 32) | operation3dword0;
flushStream();
expectMemory<FamilyType>(reinterpret_cast<void *>(expectedGpuAddressDwordOp1), &operation1dword0, sizeof(operation1dword0));
expectMemory<FamilyType>(reinterpret_cast<void *>(expectedGpuAddressDwordOp2), &operation2dword0, sizeof(operation2dword0));
expectMemory<FamilyType>(reinterpret_cast<void *>(expectedGpuAddressQwordOp3), &operation3qword, sizeof(operation3qword));
}
HWTEST_F(MiAtomicAubTest, GivenSystemMemoryWhenDispatchingAtomicMove4BytesOperationThenExpectCorrectEndValues) {
using MI_ATOMIC = typename FamilyType::MI_ATOMIC;
auto atomicAddress = systemSurface->getGpuAddress();
auto expectedGpuAddressDwordOp1 = atomicAddress;
auto expectedGpuAddressDwordOp2 = expectedGpuAddressDwordOp1 + sizeof(uint32_t);
uint32_t operation1dword0 = 0x15;
EncodeAtomic<FamilyType>::programMiAtomic(taskStream, expectedGpuAddressDwordOp1,
MI_ATOMIC::ATOMIC_OPCODES::ATOMIC_4B_MOVE,
MI_ATOMIC::DATA_SIZE::DATA_SIZE_DWORD,
0, 0, operation1dword0, 0u);
uint32_t operation2dword0 = 0xFF;
EncodeAtomic<FamilyType>::programMiAtomic(taskStream, expectedGpuAddressDwordOp2,
MI_ATOMIC::ATOMIC_OPCODES::ATOMIC_4B_MOVE,
MI_ATOMIC::DATA_SIZE::DATA_SIZE_DWORD,
0, 0, operation2dword0, 0u);
flushStream();
expectMemory<FamilyType>(reinterpret_cast<void *>(expectedGpuAddressDwordOp1), &operation1dword0, sizeof(operation1dword0));
expectMemory<FamilyType>(reinterpret_cast<void *>(expectedGpuAddressDwordOp2), &operation2dword0, sizeof(operation2dword0));
}

2
opencl/test/unit_test/mocks/mock_device_queue.h
View File

@@ -115,7 +115,7 @@ class MockDeviceQueueHw : public DeviceQueueHw<GfxFamily> {
placeholder,
MI_ATOMIC::ATOMIC_OPCODES::ATOMIC_8B_INCREMENT,
MI_ATOMIC::DATA_SIZE::DATA_SIZE_QWORD,
0x1u, 0x1u);
0x1u, 0x1u, 0x0u, 0x0u);
return miAtomic;
}

11
shared/source/command_container/command_encoder.h
View File

@@ -318,14 +318,21 @@ struct EncodeAtomic {
ATOMIC_OPCODES opcode,
DATA_SIZE dataSize,
uint32_t returnDataControl,
uint32_t csStall);
uint32_t csStall,
uint32_t operand1dword0,
uint32_t operand1dword1);
static void programMiAtomic(MI_ATOMIC *atomic,
uint64_t writeAddress,
ATOMIC_OPCODES opcode,
DATA_SIZE dataSize,
uint32_t returnDataControl,
uint32_t csStall);
uint32_t csStall,
uint32_t operand1dword0,
uint32_t operand1dword1);
static void setMiAtomicAddress(MI_ATOMIC &atomic, uint64_t writeAddress);
static uint64_t getMiAtomicAddress(MI_ATOMIC &atomic);
};
template <typename GfxFamily>

32
shared/source/command_container/command_encoder.inl
View File

@@ -569,20 +569,40 @@ size_t EncodeSempahore<Family>::getSizeMiSemaphoreWait() {
return sizeof(MI_SEMAPHORE_WAIT);
}
template <typename Family>
void EncodeAtomic<Family>::setMiAtomicAddress(MI_ATOMIC &atomic, uint64_t writeAddress) {
atomic.setMemoryAddress(static_cast<uint32_t>(writeAddress & 0x0000FFFFFFFFULL));
atomic.setMemoryAddressHigh(static_cast<uint32_t>(writeAddress >> 32));
}
template <typename Family>
uint64_t EncodeAtomic<Family>::getMiAtomicAddress(MI_ATOMIC &atomic) {
uint64_t address = (static_cast<uint64_t>(atomic.getMemoryAddressHigh()) << 32) | (atomic.getMemoryAddress());
return address;
}
template <typename Family>
void EncodeAtomic<Family>::programMiAtomic(MI_ATOMIC *atomic,
uint64_t writeAddress,
ATOMIC_OPCODES opcode,
DATA_SIZE dataSize,
uint32_t returnDataControl,
uint32_t csStall) {
uint32_t csStall,
uint32_t operand1dword0,
uint32_t operand1dword1) {
MI_ATOMIC cmd = Family::cmdInitAtomic;
cmd.setAtomicOpcode(opcode);
cmd.setDataSize(dataSize);
cmd.setMemoryAddress(static_cast<uint32_t>(writeAddress & 0x0000FFFFFFFFULL));
cmd.setMemoryAddressHigh(static_cast<uint32_t>(writeAddress >> 32));
EncodeAtomic<Family>::setMiAtomicAddress(cmd, writeAddress);
cmd.setReturnDataControl(returnDataControl);
cmd.setCsStall(csStall);
if (opcode == ATOMIC_OPCODES::ATOMIC_4B_MOVE ||
opcode == ATOMIC_OPCODES::ATOMIC_8B_MOVE) {
cmd.setDwordLength(MI_ATOMIC::DWORD_LENGTH::DWORD_LENGTH_INLINE_DATA_1);
cmd.setInlineData(0x1);
cmd.setOperand1DataDword0(operand1dword0);
cmd.setOperand1DataDword1(operand1dword1);
}
*atomic = cmd;
}
@@ -593,9 +613,11 @@ void EncodeAtomic<Family>::programMiAtomic(LinearStream &commandStream,
ATOMIC_OPCODES opcode,
DATA_SIZE dataSize,
uint32_t returnDataControl,
uint32_t csStall) {
uint32_t csStall,
uint32_t operand1dword0,
uint32_t operand1dword1) {
auto miAtomic = commandStream.getSpaceForCmd<MI_ATOMIC>();
EncodeAtomic<Family>::programMiAtomic(miAtomic, writeAddress, opcode, dataSize, returnDataControl, csStall);
EncodeAtomic<Family>::programMiAtomic(miAtomic, writeAddress, opcode, dataSize, returnDataControl, csStall, operand1dword0, operand1dword1);
}
template <typename Family>

1
shared/source/generated/gen11/hw_cmds_generated_gen11.inl
View File

@@ -870,6 +870,7 @@ typedef struct tagMI_ATOMIC {
ATOMIC_4B_MOVE = 0x4,
ATOMIC_4B_INCREMENT = 0x5,
ATOMIC_4B_DECREMENT = 0x6,
ATOMIC_8B_MOVE = 0x24,
ATOMIC_8B_INCREMENT = 0x25,
ATOMIC_8B_DECREMENT = 0x26,
} ATOMIC_OPCODES;

1
shared/source/generated/gen12lp/hw_cmds_generated_gen12lp.inl
View File

@@ -907,6 +907,7 @@ typedef struct tagMI_ATOMIC {
ATOMIC_4B_MOVE = 0x4,
ATOMIC_4B_INCREMENT = 0x5,
ATOMIC_4B_DECREMENT = 0x6,
ATOMIC_8B_MOVE = 0x24,
ATOMIC_8B_INCREMENT = 0x25,
ATOMIC_8B_DECREMENT = 0x26,
} ATOMIC_OPCODES;

1
shared/source/generated/gen8/hw_cmds_generated_gen8.inl
View File

@@ -1008,6 +1008,7 @@ typedef struct tagMI_ATOMIC {
ATOMIC_4B_MOVE = 0x4,
ATOMIC_4B_INCREMENT = 0x5,
ATOMIC_4B_DECREMENT = 0x6,
ATOMIC_8B_MOVE = 0x24,
ATOMIC_8B_INCREMENT = 0x25,
ATOMIC_8B_DECREMENT = 0x26,
} ATOMIC_OPCODES;

1
shared/source/generated/gen9/hw_cmds_generated_gen9.inl
View File

@@ -1005,6 +1005,7 @@ typedef struct tagMI_ATOMIC {
ATOMIC_4B_MOVE = 0x4,
ATOMIC_4B_INCREMENT = 0x5,
ATOMIC_4B_DECREMENT = 0x6,
ATOMIC_8B_MOVE = 0x24,
ATOMIC_8B_INCREMENT = 0x25,
ATOMIC_8B_DECREMENT = 0x26,
} ATOMIC_OPCODES;

2
shared/source/helpers/timestamp_packet.h
View File

@@ -178,7 +178,7 @@ struct TimestampPacketHelper {
EncodeAtomic<GfxFamily>::programMiAtomic(cmdStream, dependenciesCountAddress,
MI_ATOMIC::ATOMIC_OPCODES::ATOMIC_4B_INCREMENT,
MI_ATOMIC::DATA_SIZE::DATA_SIZE_DWORD,
0u, 0u);
0u, 0u, 0x0u, 0x0u);
}
}

61
shared/test/unit_test/encoders/test_encode_atomic.cpp
View File

@@ -21,21 +21,74 @@ HWTEST_F(CommandEncodeAtomic, WhenProgrammingMiAtomicThenExpectAllFieldsSetCorre
constexpr size_t bufferSize = 128u;
uint8_t buffer[bufferSize];
uint64_t address = static_cast<uint64_t>(0x123400);
LinearStream cmdbuffer(buffer, bufferSize);
EncodeAtomic<FamilyType>::programMiAtomic(cmdbuffer,
static_cast<uint64_t>(0x123400),
address,
ATOMIC_OPCODES::ATOMIC_4B_DECREMENT,
DATA_SIZE::DATA_SIZE_DWORD,
0x1u,
0x1u);
0x1u,
0x0u,
0x0u);
MI_ATOMIC *miAtomicCmd = reinterpret_cast<MI_ATOMIC *>(cmdbuffer.getCpuBase());
EXPECT_EQ(ATOMIC_OPCODES::ATOMIC_4B_DECREMENT, miAtomicCmd->getAtomicOpcode());
EXPECT_EQ(DATA_SIZE::DATA_SIZE_DWORD, miAtomicCmd->getDataSize());
EXPECT_EQ(0x123400u, miAtomicCmd->getMemoryAddress());
EXPECT_EQ(address, EncodeAtomic<FamilyType>::getMiAtomicAddress(*miAtomicCmd));
EXPECT_EQ(0x1u, miAtomicCmd->getReturnDataControl());
EXPECT_EQ(0x1u, miAtomicCmd->getCsStall());
}
HWTEST_F(CommandEncodeAtomic, WhenProgrammingMiAtomicMoveOperationThenExpectInlineDataSet) {
using MI_ATOMIC = typename FamilyType::MI_ATOMIC;
using ATOMIC_OPCODES = typename FamilyType::MI_ATOMIC::ATOMIC_OPCODES;
using DATA_SIZE = typename FamilyType::MI_ATOMIC::DATA_SIZE;
using DWORD_LENGTH = typename FamilyType::MI_ATOMIC::DWORD_LENGTH;
constexpr size_t bufferSize = 128u;
uint8_t buffer[bufferSize];
uint64_t address = (static_cast<uint64_t>(3) << 32) + 0x123400;
LinearStream cmdbuffer(buffer, bufferSize);
EncodeAtomic<FamilyType>::programMiAtomic(cmdbuffer,
address,
ATOMIC_OPCODES::ATOMIC_4B_MOVE,
DATA_SIZE::DATA_SIZE_DWORD,
0x0u,
0x0u,
0x10u,
0x20u);
EncodeAtomic<FamilyType>::programMiAtomic(cmdbuffer,
address,
ATOMIC_OPCODES::ATOMIC_8B_MOVE,
DATA_SIZE::DATA_SIZE_QWORD,
0x0u,
0x0u,
0x20u,
0x1fu);
MI_ATOMIC *miAtomicCmd = reinterpret_cast<MI_ATOMIC *>(cmdbuffer.getCpuBase());
EXPECT_EQ(ATOMIC_OPCODES::ATOMIC_4B_MOVE, miAtomicCmd->getAtomicOpcode());
EXPECT_EQ(DATA_SIZE::DATA_SIZE_DWORD, miAtomicCmd->getDataSize());
EXPECT_EQ(address, EncodeAtomic<FamilyType>::getMiAtomicAddress(*miAtomicCmd));
EXPECT_EQ(0x0u, miAtomicCmd->getReturnDataControl());
EXPECT_EQ(DWORD_LENGTH::DWORD_LENGTH_INLINE_DATA_1, miAtomicCmd->getDwordLength());
EXPECT_EQ(0x1u, miAtomicCmd->getInlineData());
EXPECT_EQ(0x10u, miAtomicCmd->getOperand1DataDword0());
EXPECT_EQ(0x20u, miAtomicCmd->getOperand1DataDword1());
miAtomicCmd++;
EXPECT_EQ(ATOMIC_OPCODES::ATOMIC_8B_MOVE, miAtomicCmd->getAtomicOpcode());
EXPECT_EQ(DATA_SIZE::DATA_SIZE_QWORD, miAtomicCmd->getDataSize());
EXPECT_EQ(address, EncodeAtomic<FamilyType>::getMiAtomicAddress(*miAtomicCmd));
EXPECT_EQ(0x0u, miAtomicCmd->getReturnDataControl());
EXPECT_EQ(DWORD_LENGTH::DWORD_LENGTH_INLINE_DATA_1, miAtomicCmd->getDwordLength());
EXPECT_EQ(0x1u, miAtomicCmd->getInlineData());
EXPECT_EQ(0x20u, miAtomicCmd->getOperand1DataDword0());
EXPECT_EQ(0x1fu, miAtomicCmd->getOperand1DataDword1());
}