compute-runtime/shared/source/direct_submission/direct_submission_hw.inl

497 lines
20 KiB
C++

/*
* Copyright (C) 2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/command_container/command_encoder.h"
#include "shared/source/command_stream/submissions_aggregator.h"
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/device/device.h"
#include "shared/source/direct_submission/direct_submission_hw.h"
#include "shared/source/direct_submission/direct_submission_hw_diagnostic_mode.h"
#include "shared/source/helpers/flush_stamp.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/memory_manager/allocation_properties.h"
#include "shared/source/memory_manager/graphics_allocation.h"
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/source/memory_manager/memory_operations_handler.h"
#include "shared/source/os_interface/os_context.h"
#include "shared/source/utilities/cpu_info.h"
#include "shared/source/utilities/cpuintrinsics.h"
#include <cstring>
namespace NEO {
template <typename GfxFamily, typename Dispatcher>
DirectSubmissionHw<GfxFamily, Dispatcher>::DirectSubmissionHw(Device &device,
OsContext &osContext)
: device(device), osContext(osContext) {
UNRECOVERABLE_IF(!CpuInfo::getInstance().isFeatureSupported(CpuInfo::featureClflush));
disableCacheFlush = UllsDefaults::defaultDisableCacheFlush;
disableMonitorFence = UllsDefaults::defaultDisableMonitorFence;
if (DebugManager.flags.DirectSubmissionDisableCacheFlush.get() != -1) {
disableCacheFlush = !!DebugManager.flags.DirectSubmissionDisableCacheFlush.get();
}
int32_t disableCacheFlushKey = DebugManager.flags.DirectSubmissionDisableCpuCacheFlush.get();
if (disableCacheFlushKey != -1) {
disableCpuCacheFlush = disableCacheFlushKey == 1 ? true : false;
}
hwInfo = &device.getHardwareInfo();
createDiagnostic();
}
template <typename GfxFamily, typename Dispatcher>
DirectSubmissionHw<GfxFamily, Dispatcher>::~DirectSubmissionHw() = default;
template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::allocateResources() {
DirectSubmissionAllocations allocations;
bool isMultiOsContextCapable = osContext.getNumSupportedDevices() > 1u;
MemoryManager *memoryManager = device.getExecutionEnvironment()->memoryManager.get();
constexpr size_t minimumRequiredSize = 256 * MemoryConstants::kiloByte;
constexpr size_t additionalAllocationSize = MemoryConstants::pageSize;
const auto allocationSize = alignUp(minimumRequiredSize + additionalAllocationSize, MemoryConstants::pageSize64k);
const AllocationProperties commandStreamAllocationProperties{device.getRootDeviceIndex(),
true, allocationSize,
GraphicsAllocation::AllocationType::RING_BUFFER,
isMultiOsContextCapable, osContext.getDeviceBitfield()};
ringBuffer = memoryManager->allocateGraphicsMemoryWithProperties(commandStreamAllocationProperties);
UNRECOVERABLE_IF(ringBuffer == nullptr);
allocations.push_back(ringBuffer);
ringBuffer2 = memoryManager->allocateGraphicsMemoryWithProperties(commandStreamAllocationProperties);
UNRECOVERABLE_IF(ringBuffer2 == nullptr);
allocations.push_back(ringBuffer2);
const AllocationProperties semaphoreAllocationProperties{device.getRootDeviceIndex(),
true, MemoryConstants::pageSize,
GraphicsAllocation::AllocationType::SEMAPHORE_BUFFER,
isMultiOsContextCapable, osContext.getDeviceBitfield()};
semaphores = memoryManager->allocateGraphicsMemoryWithProperties(semaphoreAllocationProperties);
UNRECOVERABLE_IF(semaphores == nullptr);
allocations.push_back(semaphores);
handleResidency();
ringCommandStream.replaceBuffer(ringBuffer->getUnderlyingBuffer(), minimumRequiredSize);
ringCommandStream.replaceGraphicsAllocation(ringBuffer);
memset(ringBuffer->getUnderlyingBuffer(), 0, allocationSize);
memset(ringBuffer2->getUnderlyingBuffer(), 0, allocationSize);
semaphorePtr = semaphores->getUnderlyingBuffer();
semaphoreGpuVa = semaphores->getGpuAddress();
semaphoreData = static_cast<volatile RingSemaphoreData *>(semaphorePtr);
memset(semaphorePtr, 0, sizeof(RingSemaphoreData));
semaphoreData->QueueWorkCount = 0;
cpuCachelineFlush(semaphorePtr, MemoryConstants::cacheLineSize);
workloadModeOneStoreAddress = static_cast<volatile void *>(&semaphoreData->DiagnosticModeCounter);
*static_cast<volatile uint32_t *>(workloadModeOneStoreAddress) = 0u;
auto ret = makeResourcesResident(allocations);
return ret && allocateOsResources();
}
template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::makeResourcesResident(DirectSubmissionAllocations &allocations) {
auto memoryInterface = this->device.getRootDeviceEnvironment().memoryOperationsInterface.get();
auto ret = memoryInterface->makeResident(&device, ArrayRef<GraphicsAllocation *>(allocations)) == MemoryOperationsStatus::SUCCESS;
return ret;
}
template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::cpuCachelineFlush(void *ptr, size_t size) {
if (disableCpuCacheFlush) {
return;
}
constexpr size_t cachlineBit = 6;
static_assert(MemoryConstants::cacheLineSize == 1 << cachlineBit, "cachlineBit has invalid value");
char *flushPtr = reinterpret_cast<char *>(ptr);
char *flushEndPtr = reinterpret_cast<char *>(ptr) + size;
flushPtr = alignDown(flushPtr, MemoryConstants::cacheLineSize);
flushEndPtr = alignUp(flushEndPtr, MemoryConstants::cacheLineSize);
size_t cachelines = (flushEndPtr - flushPtr) >> cachlineBit;
for (size_t i = 0; i < cachelines; i++) {
CpuIntrinsics::clFlush(flushPtr);
flushPtr += MemoryConstants::cacheLineSize;
}
}
template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::initialize(bool submitOnInit) {
bool ret = allocateResources();
initDiagnostic(submitOnInit);
if (ret && submitOnInit) {
size_t startBufferSize = Dispatcher::getSizePreemption() +
getSizeSemaphoreSection();
Dispatcher::dispatchPreemption(ringCommandStream);
if (workloadMode == 1) {
dispatchDiagnosticModeSection();
startBufferSize += getDiagnosticModeSection();
}
dispatchSemaphoreSection(currentQueueWorkCount);
ringStart = submit(ringCommandStream.getGraphicsAllocation()->getGpuAddress(), startBufferSize);
performDiagnosticMode();
return ringStart;
}
return ret;
}
template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::startRingBuffer() {
if (ringStart) {
return true;
}
size_t startSize = getSizeSemaphoreSection();
size_t requiredSize = startSize + getSizeDispatch() + getSizeEnd();
if (ringCommandStream.getAvailableSpace() < requiredSize) {
switchRingBuffers();
}
uint64_t gpuStartVa = getCommandBufferPositionGpuAddress(ringCommandStream.getSpace(0));
currentQueueWorkCount++;
dispatchSemaphoreSection(currentQueueWorkCount);
ringStart = submit(gpuStartVa, startSize);
return ringStart;
}
template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::stopRingBuffer() {
void *flushPtr = ringCommandStream.getSpace(0);
Dispatcher::dispatchCacheFlush(ringCommandStream, *hwInfo);
if (disableMonitorFence) {
TagData currentTagData = {};
getTagAddressValue(currentTagData);
Dispatcher::dispatchMonitorFence(ringCommandStream, currentTagData.tagAddress, currentTagData.tagValue, *hwInfo);
}
Dispatcher::dispatchStopCommandBuffer(ringCommandStream);
cpuCachelineFlush(flushPtr, getSizeEnd());
semaphoreData->QueueWorkCount = currentQueueWorkCount;
cpuCachelineFlush(semaphorePtr, MemoryConstants::cacheLineSize);
return true;
}
template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchSemaphoreSection(uint32_t value) {
using MI_SEMAPHORE_WAIT = typename GfxFamily::MI_SEMAPHORE_WAIT;
using COMPARE_OPERATION = typename GfxFamily::MI_SEMAPHORE_WAIT::COMPARE_OPERATION;
dispatchDisablePrefetcher(true);
EncodeSempahore<GfxFamily>::addMiSemaphoreWaitCommand(ringCommandStream,
semaphoreGpuVa,
value,
COMPARE_OPERATION::COMPARE_OPERATION_SAD_GREATER_THAN_OR_EQUAL_SDD);
dispatchPrefetchMitigation();
dispatchDisablePrefetcher(false);
}
template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeSemaphoreSection() {
size_t semaphoreSize = EncodeSempahore<GfxFamily>::getSizeMiSemaphoreWait();
semaphoreSize += getSizePrefetchMitigation();
semaphoreSize += 2 * getSizeDisablePrefetcher();
return semaphoreSize;
}
template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchStartSection(uint64_t gpuStartAddress) {
Dispatcher::dispatchStartCommandBuffer(ringCommandStream, gpuStartAddress);
}
template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeStartSection() {
return Dispatcher::getSizeStartCommandBuffer();
}
template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchSwitchRingBufferSection(uint64_t nextBufferGpuAddress) {
Dispatcher::dispatchStartCommandBuffer(ringCommandStream, nextBufferGpuAddress);
}
template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeSwitchRingBufferSection() {
return Dispatcher::getSizeStartCommandBuffer();
}
template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeEnd() {
size_t size = Dispatcher::getSizeStopCommandBuffer() +
Dispatcher::getSizeCacheFlush(*hwInfo);
if (disableMonitorFence) {
size += Dispatcher::getSizeMonitorFence(*hwInfo);
}
return size;
}
template <typename GfxFamily, typename Dispatcher>
inline uint64_t DirectSubmissionHw<GfxFamily, Dispatcher>::getCommandBufferPositionGpuAddress(void *position) {
void *currentBase = ringCommandStream.getCpuBase();
size_t offset = ptrDiff(position, currentBase);
return ringCommandStream.getGraphicsAllocation()->getGpuAddress() + static_cast<uint64_t>(offset);
}
template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeDispatch() {
size_t size = getSizeSemaphoreSection();
if (workloadMode == 0) {
size += getSizeStartSection();
} else if (workloadMode == 1) {
size += getDiagnosticModeSection();
}
//mode 2 does not dispatch any commands
if (!disableCacheFlush) {
size += Dispatcher::getSizeCacheFlush(*hwInfo);
}
if (!disableMonitorFence) {
size += Dispatcher::getSizeMonitorFence(*hwInfo);
}
return size;
}
template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchPrefetchMitigation() {
uint32_t *prefetchNoop = static_cast<uint32_t *>(ringCommandStream.getSpace(prefetchSize));
size_t i = 0u;
while (i < prefetchNoops) {
*prefetchNoop = 0u;
prefetchNoop++;
i++;
}
}
template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizePrefetchMitigation() {
return prefetchSize;
}
template <typename GfxFamily, typename Dispatcher>
void *DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchWorkloadSection(BatchBuffer &batchBuffer) {
void *currentPosition = ringCommandStream.getSpace(0);
if (workloadMode == 0) {
auto commandStreamAddress = ptrOffset(batchBuffer.commandBufferAllocation->getGpuAddress(), batchBuffer.startOffset);
void *returnCmd = batchBuffer.endCmdPtr;
dispatchStartSection(commandStreamAddress);
void *returnPosition = ringCommandStream.getSpace(0);
setReturnAddress(returnCmd, getCommandBufferPositionGpuAddress(returnPosition));
} else if (workloadMode == 1) {
DirectSubmissionDiagnostics::diagnosticModeOneDispatch(diagnostic.get());
dispatchDiagnosticModeSection();
}
//mode 2 does not dispatch any commands
if (!disableCacheFlush) {
Dispatcher::dispatchCacheFlush(ringCommandStream, *hwInfo);
}
if (!disableMonitorFence) {
TagData currentTagData = {};
getTagAddressValue(currentTagData);
Dispatcher::dispatchMonitorFence(ringCommandStream, currentTagData.tagAddress, currentTagData.tagValue, *hwInfo);
}
dispatchSemaphoreSection(currentQueueWorkCount + 1);
return currentPosition;
}
template <typename GfxFamily, typename Dispatcher>
bool DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchCommandBuffer(BatchBuffer &batchBuffer, FlushStampTracker &flushStamp) {
//for now workloads requiring cache coherency are not supported
UNRECOVERABLE_IF(batchBuffer.requiresCoherency);
size_t dispatchSize = getSizeDispatch();
size_t cycleSize = getSizeSwitchRingBufferSection();
size_t requiredMinimalSize = dispatchSize + cycleSize + getSizeEnd();
bool buffersSwitched = false;
uint64_t startGpuVa = getCommandBufferPositionGpuAddress(ringCommandStream.getSpace(0));
if (ringCommandStream.getAvailableSpace() < requiredMinimalSize) {
startGpuVa = switchRingBuffers();
buffersSwitched = true;
}
void *currentPosition = dispatchWorkloadSection(batchBuffer);
if (ringStart) {
cpuCachelineFlush(currentPosition, dispatchSize);
handleResidency();
}
//unblock GPU
semaphoreData->QueueWorkCount = currentQueueWorkCount;
cpuCachelineFlush(semaphorePtr, MemoryConstants::cacheLineSize);
currentQueueWorkCount++;
DirectSubmissionDiagnostics::diagnosticModeOneSubmit(diagnostic.get());
//when ring buffer is not started at init or being restarted
if (!ringStart) {
size_t submitSize = dispatchSize;
if (buffersSwitched) {
submitSize = cycleSize;
}
ringStart = submit(startGpuVa, submitSize);
}
uint64_t flushValue = updateTagValue();
flushStamp.setStamp(flushValue);
return ringStart;
}
template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::setReturnAddress(void *returnCmd, uint64_t returnAddress) {
using MI_BATCH_BUFFER_START = typename GfxFamily::MI_BATCH_BUFFER_START;
MI_BATCH_BUFFER_START cmd = GfxFamily::cmdInitBatchBufferStart;
cmd.setBatchBufferStartAddressGraphicsaddress472(returnAddress);
cmd.setAddressSpaceIndicator(MI_BATCH_BUFFER_START::ADDRESS_SPACE_INDICATOR_PPGTT);
MI_BATCH_BUFFER_START *returnBBStart = static_cast<MI_BATCH_BUFFER_START *>(returnCmd);
*returnBBStart = cmd;
}
template <typename GfxFamily, typename Dispatcher>
inline void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchDisablePrefetcher(bool disable) {
}
template <typename GfxFamily, typename Dispatcher>
inline size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getSizeDisablePrefetcher() {
return 0u;
}
template <typename GfxFamily, typename Dispatcher>
inline uint64_t DirectSubmissionHw<GfxFamily, Dispatcher>::switchRingBuffers() {
GraphicsAllocation *nextRingBuffer = switchRingBuffersAllocations();
void *flushPtr = ringCommandStream.getSpace(0);
uint64_t currentBufferGpuVa = getCommandBufferPositionGpuAddress(flushPtr);
if (ringStart) {
dispatchSwitchRingBufferSection(nextRingBuffer->getGpuAddress());
cpuCachelineFlush(flushPtr, getSizeSwitchRingBufferSection());
}
ringCommandStream.replaceBuffer(nextRingBuffer->getUnderlyingBuffer(), ringCommandStream.getMaxAvailableSpace());
ringCommandStream.replaceGraphicsAllocation(nextRingBuffer);
handleSwitchRingBuffers();
return currentBufferGpuVa;
}
template <typename GfxFamily, typename Dispatcher>
inline GraphicsAllocation *DirectSubmissionHw<GfxFamily, Dispatcher>::switchRingBuffersAllocations() {
GraphicsAllocation *nextAllocation = nullptr;
if (currentRingBuffer == RingBufferUse::FirstBuffer) {
nextAllocation = ringBuffer2;
currentRingBuffer = RingBufferUse::SecondBuffer;
} else {
nextAllocation = ringBuffer;
currentRingBuffer = RingBufferUse::FirstBuffer;
}
return nextAllocation;
}
template <typename GfxFamily, typename Dispatcher>
void DirectSubmissionHw<GfxFamily, Dispatcher>::deallocateResources() {
MemoryManager *memoryManager = device.getExecutionEnvironment()->memoryManager.get();
if (ringBuffer) {
memoryManager->freeGraphicsMemory(ringBuffer);
ringBuffer = nullptr;
}
if (ringBuffer2) {
memoryManager->freeGraphicsMemory(ringBuffer2);
ringBuffer2 = nullptr;
}
if (semaphores) {
memoryManager->freeGraphicsMemory(semaphores);
semaphores = nullptr;
}
}
template <typename GfxFamily, typename Dispatcher>
void DirectSubmissionHw<GfxFamily, Dispatcher>::createDiagnostic() {
if (directSubmissionDiagnosticAvailable) {
workloadMode = DebugManager.flags.DirectSubmissionEnableDebugBuffer.get();
if (workloadMode > 0) {
disableCacheFlush = DebugManager.flags.DirectSubmissionDisableCacheFlush.get();
disableMonitorFence = DebugManager.flags.DirectSubmissionDisableMonitorFence.get();
uint32_t executions = static_cast<uint32_t>(DebugManager.flags.DirectSubmissionDiagnosticExecutionCount.get());
diagnostic = std::make_unique<DirectSubmissionDiagnosticsCollector>(
executions,
workloadMode == 1,
DebugManager.flags.DirectSubmissionBufferPlacement.get(),
DebugManager.flags.DirectSubmissionSemaphorePlacement.get(),
workloadMode,
disableCacheFlush,
disableMonitorFence);
}
}
}
template <typename GfxFamily, typename Dispatcher>
void DirectSubmissionHw<GfxFamily, Dispatcher>::initDiagnostic(bool &submitOnInit) {
if (directSubmissionDiagnosticAvailable) {
if (diagnostic.get()) {
submitOnInit = true;
diagnostic->diagnosticModeAllocation();
}
}
}
template <typename GfxFamily, typename Dispatcher>
void DirectSubmissionHw<GfxFamily, Dispatcher>::performDiagnosticMode() {
if (directSubmissionDiagnosticAvailable) {
if (diagnostic.get()) {
diagnostic->diagnosticModeDiagnostic();
if (workloadMode == 1) {
diagnostic->diagnosticModeOneWait(workloadModeOneStoreAddress, workloadModeOneExpectedValue);
}
BatchBuffer dummyBuffer = {};
FlushStampTracker dummyTracker(true);
for (uint32_t execution = 0; execution < diagnostic->getExecutionsCount(); execution++) {
dispatchCommandBuffer(dummyBuffer, dummyTracker);
if (workloadMode == 1) {
diagnostic->diagnosticModeOneWaitCollect(execution, workloadModeOneStoreAddress, workloadModeOneExpectedValue);
}
}
workloadMode = 0;
disableCacheFlush = UllsDefaults::defaultDisableCacheFlush;
disableMonitorFence = UllsDefaults::defaultDisableMonitorFence;
diagnostic.reset(nullptr);
}
}
}
template <typename GfxFamily, typename Dispatcher>
void DirectSubmissionHw<GfxFamily, Dispatcher>::dispatchDiagnosticModeSection() {
workloadModeOneExpectedValue++;
uint64_t storeAddress = semaphoreGpuVa;
storeAddress += ptrDiff(workloadModeOneStoreAddress, semaphorePtr);
Dispatcher::dispatchStoreDwordCommand(ringCommandStream, storeAddress, workloadModeOneExpectedValue);
}
template <typename GfxFamily, typename Dispatcher>
size_t DirectSubmissionHw<GfxFamily, Dispatcher>::getDiagnosticModeSection() {
return Dispatcher::getSizeStoreDwordCommand();
}
} // namespace NEO