Media
/
compute-runtime
mirror of https://github.com/intel/compute-runtime.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
compute-runtime/shared/source/os_interface/linux/drm_neo.cpp

1872 lines
73 KiB
C++
Raw Blame History

/*
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/os_interface/linux/drm_neo.h"
#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/command_stream/submission_status.h"
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/execution_environment/execution_environment.h"
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/gmm_helper/cache_settings_helper.h"
#include "shared/source/gmm_helper/client_context/gmm_client_context.h"
#include "shared/source/gmm_helper/gmm.h"
#include "shared/source/gmm_helper/resource_info.h"
#include "shared/source/helpers/basic_math.h"
#include "shared/source/helpers/constants.h"
#include "shared/source/helpers/debug_helpers.h"
#include "shared/source/helpers/gfx_core_helper.h"
#include "shared/source/helpers/gpu_page_fault_helper.h"
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/os_interface/driver_info.h"
#include "shared/source/os_interface/linux/cache_info.h"
#include "shared/source/os_interface/linux/drm_buffer_object.h"
#include "shared/source/os_interface/linux/drm_engine_mapper.h"
#include "shared/source/os_interface/linux/drm_gem_close_worker.h"
#include "shared/source/os_interface/linux/drm_memory_manager.h"
#include "shared/source/os_interface/linux/drm_memory_operations_handler_bind.h"
#include "shared/source/os_interface/linux/drm_wrappers.h"
#include "shared/source/os_interface/linux/engine_info.h"
#include "shared/source/os_interface/linux/hw_device_id.h"
#include "shared/source/os_interface/linux/ioctl_helper.h"
#include "shared/source/os_interface/linux/memory_info.h"
#include "shared/source/os_interface/linux/os_context_linux.h"
#include "shared/source/os_interface/linux/os_inc.h"
#include "shared/source/os_interface/linux/pci_path.h"
#include "shared/source/os_interface/linux/sys_calls.h"
#include "shared/source/os_interface/linux/system_info.h"
#include "shared/source/os_interface/linux/xe/ioctl_helper_xe.h"
#include "shared/source/os_interface/os_environment.h"
#include "shared/source/os_interface/os_interface.h"
#include "shared/source/os_interface/product_helper.h"
#include "shared/source/release_helper/release_helper.h"
#include "shared/source/utilities/api_intercept.h"
#include "shared/source/utilities/cpu_info.h"
#include "shared/source/utilities/directory.h"
#include "shared/source/utilities/io_functions.h"
#include "xe_drm.h"
#include <cstdio>
#include <cstring>
#include <fcntl.h>
#include <fstream>
#include <map>
#include <sstream>
namespace NEO {
Drm::Drm(std::unique_ptr<HwDeviceIdDrm> &&hwDeviceIdIn, RootDeviceEnvironment &rootDeviceEnvironment)
: DriverModel(DriverModelType::drm),
hwDeviceId(std::move(hwDeviceIdIn)), rootDeviceEnvironment(rootDeviceEnvironment) {
pagingFence.fill(0u);
fenceVal.fill(0u);
}
SubmissionStatus Drm::getSubmissionStatusFromReturnCode(int32_t retCode) {
switch (retCode) {
case 0:
return SubmissionStatus::success;
case EWOULDBLOCK:
case ENOMEM:
case ENOSPC:
return SubmissionStatus::outOfHostMemory;
case ENXIO:
return SubmissionStatus::outOfMemory;
default:
return SubmissionStatus::failed;
}
}
void Drm::queryAndSetVmBindPatIndexProgrammingSupport() {
auto &productHelper = rootDeviceEnvironment.getHelper<ProductHelper>();
this->vmBindPatIndexProgrammingSupported = productHelper.isVmBindPatIndexProgrammingSupported();
}
int Drm::ioctl(DrmIoctl request, void *arg) {
auto requestValue = getIoctlRequestValue(request, ioctlHelper.get());
int ret;
int returnedErrno = 0;
SYSTEM_ENTER();
do {
auto measureTime = debugManager.flags.PrintKmdTimes.get();
std::chrono::steady_clock::time_point start;
std::chrono::steady_clock::time_point end;
auto printIoctl = debugManager.flags.PrintIoctlEntries.get();
if (printIoctl) {
printf("IOCTL %s called\n", ioctlHelper->getIoctlString(request).c_str());
}
if (measureTime) {
start = std::chrono::steady_clock::now();
}
ret = SysCalls::ioctl(getFileDescriptor(), requestValue, arg);
if (ret != 0) {
returnedErrno = getErrno();
}
if (measureTime) {
end = std::chrono::steady_clock::now();
long long elapsedTime = std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count();
static std::mutex mtx;
std::lock_guard lock(mtx);
IoctlStatisticsEntry ioctlData{};
auto ioctlDataIt = this->ioctlStatistics.find(request);
if (ioctlDataIt != this->ioctlStatistics.end()) {
ioctlData = ioctlDataIt->second;
}
ioctlData.totalTime += elapsedTime;
ioctlData.count++;
ioctlData.minTime = std::min(ioctlData.minTime, elapsedTime);
ioctlData.maxTime = std::max(ioctlData.maxTime, elapsedTime);
this->ioctlStatistics[request] = ioctlData;
}
if (printIoctl) {
if (ret == 0) {
printf("IOCTL %s returns %d\n",
ioctlHelper->getIoctlString(request).c_str(), ret);
} else {
printf("IOCTL %s returns %d, errno %d(%s)\n",
ioctlHelper->getIoctlString(request).c_str(), ret, returnedErrno, strerror(returnedErrno));
}
}
} while (ret == -1 && checkIfIoctlReinvokeRequired(returnedErrno, request, ioctlHelper.get()));
SYSTEM_LEAVE(request);
return ret;
}
int Drm::getParamIoctl(DrmParam param, int *dstValue) {
GetParam getParam{};
getParam.param = ioctlHelper->getDrmParamValue(param);
getParam.value = dstValue;
int retVal = ioctlHelper->ioctl(DrmIoctl::getparam, &getParam);
if (debugManager.flags.PrintIoctlEntries.get()) {
printf("DRM_IOCTL_I915_GETPARAM: param: %s, output value: %d, retCode:% d\n",
ioctlHelper->getDrmParamString(param).c_str(),
*getParam.value,
retVal);
}
return retVal;
}
int Drm::enableTurboBoost() {
GemContextParam contextParam = {};
contextParam.param = contextPrivateParamBoost;
contextParam.value = 1;
return ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
}
int Drm::getEnabledPooledEu(int &enabled) {
return getParamIoctl(DrmParam::paramHasPooledEu, &enabled);
}
std::string Drm::getSysFsPciPathBaseName() {
auto fullPath = getSysFsPciPath();
size_t pos = fullPath.rfind("/");
if (std::string::npos == pos) {
return fullPath;
}
return fullPath.substr(pos + 1, std::string::npos);
}
std::string Drm::getSysFsPciPath() {
std::string path = std::string(Os::sysFsPciPathPrefix) + hwDeviceId->getPciPath() + "/drm";
std::string expectedFilePrefix = path + "/card";
auto files = Directory::getFiles(path.c_str());
for (auto &file : files) {
if (file.find(expectedFilePrefix.c_str()) != std::string::npos) {
return file;
}
}
return {};
}
bool Drm::readSysFsAsString(const std::string &relativeFilePath, std::string &readString) {
auto devicePath = getSysFsPciPath();
if (devicePath.empty()) {
return false;
}
const std::string fileName = devicePath + relativeFilePath;
int fd = SysCalls::open(fileName.c_str(), O_RDONLY);
if (fd < 0) {
return false;
}
ssize_t bytesRead = SysCalls::pread(fd, readString.data(), readString.size() - 1, 0);
NEO::SysCalls::close(fd);
if (bytesRead <= 0) {
return false;
}
std::replace(readString.begin(), readString.end(), '\n', '\0');
return true;
}
int Drm::queryGttSize(uint64_t &gttSizeOutput, bool alignUpToFullRange) {
GemContextParam contextParam = {0};
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamGttSize);
int ret = ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &contextParam);
if (ret == 0) {
if (alignUpToFullRange) {
gttSizeOutput = Drm::alignUpGttSize(contextParam.value);
} else {
gttSizeOutput = contextParam.value;
}
}
return ret;
}
bool Drm::isGpuHangDetected(OsContext &osContext) {
bool ret = checkResetStatus(osContext);
auto threshold = getGpuFaultCheckThreshold();
if (checkGpuPageFaultRequired()) {
if (gpuFaultCheckCounter >= threshold) {
auto memoryManager = static_cast<DrmMemoryManager *>(this->rootDeviceEnvironment.executionEnvironment.memoryManager.get());
memoryManager->checkUnexpectedGpuPageFault();
gpuFaultCheckCounter = 0;
return false;
}
gpuFaultCheckCounter++;
}
return ret;
}
bool Drm::checkResetStatus(OsContext &osContext) {
const auto osContextLinux = static_cast<OsContextLinux *>(&osContext);
const auto &drmContextIds = osContextLinux->getDrmContextIds();
for (const auto drmContextId : drmContextIds) {
ResetStats resetStats{};
resetStats.contextId = drmContextId;
ResetStatsFault fault{};
uint32_t status = 0;
const auto retVal{ioctlHelper->getResetStats(resetStats, &status, &fault)};
UNRECOVERABLE_IF(retVal != 0);
auto debuggingEnabled = rootDeviceEnvironment.executionEnvironment.isDebuggingEnabled();
if (checkToDisableScratchPage() && ioctlHelper->validPageFault(fault.flags)) {
bool banned = ((status & ioctlHelper->getStatusForResetStats(true)) != 0);
if (!banned && debuggingEnabled) {
return false;
}
IoFunctions::fprintf(stderr, "Segmentation fault from GPU at 0x%llx, ctx_id: %u (%s) type: %d (%s), level: %d (%s), access: %d (%s), banned: %d, aborting.\n",
fault.addr,
resetStats.contextId,
EngineHelpers::engineTypeToString(osContext.getEngineType()).c_str(),
fault.type, GpuPageFaultHelpers::faultTypeToString(static_cast<FaultType>(fault.type)).c_str(),
fault.level, GpuPageFaultHelpers::faultLevelToString(static_cast<FaultLevel>(fault.level)).c_str(),
fault.access, GpuPageFaultHelpers::faultAccessToString(static_cast<FaultAccess>(fault.access)).c_str(),
banned);
IoFunctions::fprintf(stdout, "Segmentation fault from GPU at 0x%llx, ctx_id: %u (%s) type: %d (%s), level: %d (%s), access: %d (%s), banned: %d, aborting.\n",
fault.addr,
resetStats.contextId,
EngineHelpers::engineTypeToString(osContext.getEngineType()).c_str(),
fault.type, GpuPageFaultHelpers::faultTypeToString(static_cast<FaultType>(fault.type)).c_str(),
fault.level, GpuPageFaultHelpers::faultLevelToString(static_cast<FaultLevel>(fault.level)).c_str(),
fault.access, GpuPageFaultHelpers::faultAccessToString(static_cast<FaultAccess>(fault.access)).c_str(),
banned);
UNRECOVERABLE_IF(true);
}
if (resetStats.batchActive > 0 || resetStats.batchPending > 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "ERROR: GPU HANG detected!\n");
osContextLinux->setHangDetected();
return true;
}
}
return false;
}
void Drm::checkPreemptionSupport() {
preemptionSupported = ioctlHelper->isPreemptionSupported();
}
void Drm::checkQueueSliceSupport() {
sliceCountChangeSupported = getQueueSliceCount(&sseu) == 0 ? true : false;
}
void Drm::setLowPriorityContextParam(uint32_t drmContextId) {
GemContextParam gcp = {};
gcp.contextId = drmContextId;
gcp.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamPriority);
gcp.value = -1023;
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &gcp);
UNRECOVERABLE_IF(retVal != 0);
}
int Drm::getQueueSliceCount(GemContextParamSseu *sseu) {
GemContextParam contextParam = {};
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamSseu);
sseu->engine.engineClass = ioctlHelper->getDrmParamValue(DrmParam::engineClassRender);
sseu->engine.engineInstance = ioctlHelper->getDrmParamValue(DrmParam::execDefault);
contextParam.value = reinterpret_cast<uint64_t>(sseu);
contextParam.size = sizeof(struct GemContextParamSseu);
return ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &contextParam);
}
uint64_t Drm::getSliceMask(uint64_t sliceCount) {
return maxNBitValue(sliceCount);
}
bool Drm::setQueueSliceCount(uint64_t sliceCount) {
if (sliceCountChangeSupported) {
GemContextParam contextParam = {};
sseu.sliceMask = getSliceMask(sliceCount);
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamSseu);
contextParam.contextId = 0;
contextParam.value = reinterpret_cast<uint64_t>(&sseu);
contextParam.size = sizeof(struct GemContextParamSseu);
int retVal = ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
if (retVal == 0) {
return true;
}
}
return false;
}
void Drm::checkNonPersistentContextsSupport() {
GemContextParam contextParam = {};
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamPersistence);
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &contextParam);
if (retVal == 0 && contextParam.value == 1) {
nonPersistentContextsSupported = true;
} else {
nonPersistentContextsSupported = false;
}
}
void Drm::setNonPersistentContext(uint32_t drmContextId) {
GemContextParam contextParam = {};
contextParam.contextId = drmContextId;
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamPersistence);
ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
}
void Drm::setUnrecoverableContext(uint32_t drmContextId) {
GemContextParam contextParam = {};
contextParam.contextId = drmContextId;
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamRecoverable);
contextParam.value = 0;
contextParam.size = 0;
ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
}
int Drm::createDrmContext(uint32_t drmVmId, bool isDirectSubmissionRequested, bool isCooperativeContextRequested) {
GemContextCreateExt gcc{};
if (debugManager.flags.DirectSubmissionDrmContext.get() != -1) {
isDirectSubmissionRequested = debugManager.flags.DirectSubmissionDrmContext.get();
}
if (isDirectSubmissionRequested) {
gcc.flags |= ioctlHelper->getDirectSubmissionFlag();
}
GemContextCreateExtSetParam extSetparam = {};
GemContextCreateExtSetParam extSetparamLowLatency = {};
if (drmVmId > 0) {
extSetparam.base.name = ioctlHelper->getDrmParamValue(DrmParam::contextCreateExtSetparam);
extSetparam.param.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamVm);
extSetparam.param.value = drmVmId;
if (ioctlHelper->hasContextFreqHint()) {
extSetparam.base.nextExtension = reinterpret_cast<uint64_t>(&extSetparamLowLatency.base);
ioctlHelper->fillExtSetparamLowLatency(extSetparamLowLatency);
}
gcc.extensions = reinterpret_cast<uint64_t>(&extSetparam);
gcc.flags |= ioctlHelper->getDrmParamValue(DrmParam::contextCreateFlagsUseExtensions);
}
if (debugManager.flags.CreateContextWithAccessCounters.get() > 0) {
return ioctlHelper->createContextWithAccessCounters(gcc);
}
if (debugManager.flags.ForceRunAloneContext.get() != -1) {
isCooperativeContextRequested = debugManager.flags.ForceRunAloneContext.get();
}
if (isCooperativeContextRequested) {
return ioctlHelper->createCooperativeContext(gcc);
}
auto ioctlResult = ioctlHelper->ioctl(DrmIoctl::gemContextCreateExt, &gcc);
if (ioctlResult < 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "WARNING: GemContextCreateExt ioctl failed. Not exposing this root device\n");
return ioctlResult;
}
return gcc.contextId;
}
void Drm::destroyDrmContext(uint32_t drmContextId) {
GemContextDestroy destroy{};
destroy.contextId = drmContextId;
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextDestroy, &destroy);
UNRECOVERABLE_IF((retVal != 0) && (errno != ENODEV));
}
void Drm::destroyDrmVirtualMemory(uint32_t drmVmId) {
GemVmControl ctl = {};
ctl.vmId = drmVmId;
auto ret = ioctlHelper->ioctl(DrmIoctl::gemVmDestroy, &ctl);
UNRECOVERABLE_IF((ret != 0) && (errno != ENODEV));
}
int Drm::queryVmId(uint32_t drmContextId, uint32_t &vmId) {
GemContextParam param{};
param.contextId = drmContextId;
param.value = 0;
param.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamVm);
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &param);
vmId = static_cast<uint32_t>(param.value);
return retVal;
}
std::unique_lock<std::mutex> Drm::lockBindFenceMutex() {
return std::unique_lock<std::mutex>(this->bindFenceMutex);
}
int Drm::getEuTotal(int &euTotal) {
return getParamIoctl(DrmParam::paramEuTotal, &euTotal);
}
int Drm::getSubsliceTotal(int &subsliceTotal) {
return getParamIoctl(DrmParam::paramSubsliceTotal, &subsliceTotal);
}
int Drm::getMinEuInPool(int &minEUinPool) {
return getParamIoctl(DrmParam::paramMinEuInPool, &minEUinPool);
}
int Drm::getErrno() {
return errno;
}
int Drm::setupHardwareInfo(const DeviceDescriptor *device, bool setupFeatureTableAndWorkaroundTable) {
const auto usDeviceIdOverride = rootDeviceEnvironment.getHardwareInfo()->platform.usDeviceID;
const auto usRevIdOverride = rootDeviceEnvironment.getHardwareInfo()->platform.usRevId;
// reset hwInfo and apply overrides
rootDeviceEnvironment.setHwInfo(device->pHwInfo);
HardwareInfo *hwInfo = rootDeviceEnvironment.getMutableHardwareInfo();
hwInfo->platform.usDeviceID = usDeviceIdOverride;
hwInfo->platform.usRevId = usRevIdOverride;
rootDeviceEnvironment.initProductHelper();
rootDeviceEnvironment.initGfxCoreHelper();
rootDeviceEnvironment.initializeGfxCoreHelperFromProductHelper();
rootDeviceEnvironment.initApiGfxCoreHelper();
rootDeviceEnvironment.initCompilerProductHelper();
rootDeviceEnvironment.initAilConfigurationHelper();
rootDeviceEnvironment.initWaitUtils();
auto result = rootDeviceEnvironment.initAilConfiguration();
if (false == result) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: AIL creation failed!\n");
return -1;
}
const auto productFamily = hwInfo->platform.eProductFamily;
setupIoctlHelper(productFamily);
ioctlHelper->setupIpVersion();
rootDeviceEnvironment.initReleaseHelper();
auto releaseHelper = rootDeviceEnvironment.getReleaseHelper();
device->setupHardwareInfo(hwInfo, setupFeatureTableAndWorkaroundTable, releaseHelper);
querySystemInfo();
if (systemInfo) {
systemInfo->checkSysInfoMismatch(hwInfo);
setupSystemInfo(hwInfo, systemInfo.get());
auto numRegions = systemInfo->getNumRegions();
if (numRegions > 0) {
hwInfo->featureTable.regionCount = numRegions;
}
}
if (!queryMemoryInfo()) {
setPerContextVMRequired(true);
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "WARNING: Failed to query memory info\n");
} else if (getMemoryInfo()->isSmallBarDetected()) {
IoFunctions::fprintf(stderr, "WARNING: Small BAR detected for device %s\n", getPciPath().c_str());
return -1;
}
if (!queryEngineInfo()) {
setPerContextVMRequired(true);
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "WARNING: Failed to query engine info\n");
}
if (!hwInfo->gtSystemInfo.L3BankCount) {
hwInfo->gtSystemInfo.L3BankCount = hwInfo->gtSystemInfo.MaxDualSubSlicesSupported;
}
DrmQueryTopologyData topologyData = {};
if (!queryTopology(*hwInfo, topologyData)) {
topologyData.sliceCount = hwInfo->gtSystemInfo.SliceCount;
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "WARNING: Topology query failed!\n");
auto ret = getEuTotal(topologyData.euCount);
if (ret != 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Cannot query EU total parameter!\n");
return ret;
}
ret = getSubsliceTotal(topologyData.subSliceCount);
if (ret != 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Cannot query subslice total parameter!\n");
return ret;
}
}
hwInfo->gtSystemInfo.SliceCount = static_cast<uint32_t>(topologyData.sliceCount);
if (!topologyMap.empty()) {
hwInfo->gtSystemInfo.IsDynamicallyPopulated = true;
std::bitset<GT_MAX_SLICE> totalSliceMask{maxNBitValue(GT_MAX_SLICE)};
uint32_t latestSliceIndex = 0;
for (auto &mapping : topologyMap) {
std::bitset<GT_MAX_SLICE> sliceMask;
DEBUG_BREAK_IF(mapping.second.sliceIndices.empty());
for (auto &slice : mapping.second.sliceIndices) {
sliceMask.set(slice);
latestSliceIndex = slice;
}
totalSliceMask &= sliceMask;
}
for (uint32_t slice = 0; slice < GT_MAX_SLICE; slice++) {
hwInfo->gtSystemInfo.SliceInfo[slice].Enabled = totalSliceMask.test(slice);
}
if (totalSliceMask.none()) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Incorrect slice mask from topology map!\n");
return -1;
}
if (totalSliceMask.count() == 1u) {
std::bitset<GT_MAX_SUBSLICE_PER_SLICE> totalSubSliceMask{maxNBitValue(GT_MAX_SUBSLICE_PER_SLICE)};
for (auto &mapping : topologyMap) {
std::bitset<GT_MAX_SUBSLICE_PER_SLICE> subSliceMask;
DEBUG_BREAK_IF(mapping.second.subsliceIndices.empty());
for (auto &subslice : mapping.second.subsliceIndices) {
if (subslice >= GT_MAX_SUBSLICE_PER_SLICE) {
subSliceMask = {};
break;
}
subSliceMask.set(subslice);
}
totalSubSliceMask &= subSliceMask;
}
for (uint32_t subslice = 0; subslice < GT_MAX_SUBSLICE_PER_SLICE; subslice++) {
hwInfo->gtSystemInfo.SliceInfo[latestSliceIndex].SubSliceInfo[subslice].Enabled = totalSubSliceMask.test(subslice);
}
}
}
hwInfo->gtSystemInfo.SubSliceCount = static_cast<uint32_t>(topologyData.subSliceCount);
hwInfo->gtSystemInfo.DualSubSliceCount = static_cast<uint32_t>(topologyData.subSliceCount);
if (!hwInfo->gtSystemInfo.MaxEuPerSubSlice) {
hwInfo->gtSystemInfo.MaxEuPerSubSlice = topologyData.maxEusPerSubSlice;
}
auto maxEuCount = static_cast<uint32_t>(topologyData.subSliceCount) * hwInfo->gtSystemInfo.MaxEuPerSubSlice;
if (topologyData.euCount == 0 || static_cast<uint32_t>(topologyData.euCount) > maxEuCount) {
hwInfo->gtSystemInfo.EUCount = maxEuCount;
} else {
hwInfo->gtSystemInfo.EUCount = static_cast<uint32_t>(topologyData.euCount);
}
if (!hwInfo->gtSystemInfo.EUCount) {
return -1;
}
auto numThreadsPerEu = systemInfo ? systemInfo->getNumThreadsPerEu() : (releaseHelper ? releaseHelper->getNumThreadsPerEu() : 7u);
hwInfo->gtSystemInfo.ThreadCount = numThreadsPerEu * hwInfo->gtSystemInfo.EUCount;
hwInfo->gtSystemInfo.MaxSlicesSupported = hwInfo->gtSystemInfo.SliceCount;
auto calculatedMaxSubSliceCount = topologyData.maxSlices * topologyData.maxSubSlicesPerSlice;
auto maxSubSliceCount = std::max(static_cast<uint32_t>(calculatedMaxSubSliceCount), hwInfo->gtSystemInfo.MaxSubSlicesSupported);
hwInfo->gtSystemInfo.MaxSubSlicesSupported = maxSubSliceCount;
hwInfo->gtSystemInfo.MaxDualSubSlicesSupported = maxSubSliceCount;
if (topologyData.numL3Banks > 0) {
hwInfo->gtSystemInfo.L3BankCount = topologyData.numL3Banks;
}
if (systemInfo) {
hwInfo->gtSystemInfo.L3CacheSizeInKb = systemInfo->getL3BankSizeInKb() * hwInfo->gtSystemInfo.L3BankCount;
}
rootDeviceEnvironment.setRcsExposure();
setupCacheInfo(*hwInfo);
hwInfo->capabilityTable.deviceName = device->devName;
rootDeviceEnvironment.initializeGfxCoreHelperFromHwInfo();
return 0;
}
void appendHwDeviceId(std::vector<std::unique_ptr<HwDeviceId>> &hwDeviceIds, int fileDescriptor, const char *pciPath, const char *devNodePath) {
if (fileDescriptor >= 0) {
if (Drm::isDrmSupported(fileDescriptor)) {
hwDeviceIds.push_back(std::make_unique<HwDeviceIdDrm>(fileDescriptor, pciPath, devNodePath));
} else {
SysCalls::close(fileDescriptor);
}
}
}
std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevices(ExecutionEnvironment &executionEnvironment) {
std::string str = "";
return Drm::discoverDevices(executionEnvironment, str);
}
std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevice(ExecutionEnvironment &executionEnvironment, std::string &osPciPath) {
return Drm::discoverDevices(executionEnvironment, osPciPath);
}
std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevices(ExecutionEnvironment &executionEnvironment, std::string &osPciPath) {
std::vector<std::unique_ptr<HwDeviceId>> hwDeviceIds;
executionEnvironment.osEnvironment = std::make_unique<OsEnvironment>();
size_t numRootDevices = 0u;
if (debugManager.flags.CreateMultipleRootDevices.get()) {
numRootDevices = debugManager.flags.CreateMultipleRootDevices.get();
}
std::vector<std::string> files = Directory::getFiles(Os::pciDevicesDirectory);
if (files.size() == 0) {
const char *pathPrefix = "/dev/dri/renderD";
const unsigned int maxDrmDevices = 64;
unsigned int startNum = 128;
for (unsigned int i = 0; i < maxDrmDevices; i++) {
std::string path = std::string(pathPrefix) + std::to_string(i + startNum);
int fileDescriptor = SysCalls::open(path.c_str(), O_RDWR | O_CLOEXEC);
if (fileDescriptor < 0) {
continue;
}
auto pciPath = NEO::getPciPath(fileDescriptor);
appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.value_or("0000:00:02.0").c_str(), path.c_str());
if (!hwDeviceIds.empty() && hwDeviceIds.size() == numRootDevices) {
break;
}
}
return hwDeviceIds;
}
do {
const char *renderDeviceSuffix = "-render";
for (std::vector<std::string>::iterator file = files.begin(); file != files.end(); ++file) {
std::string_view devicePathView(file->c_str(), file->size());
devicePathView = devicePathView.substr(strlen(Os::pciDevicesDirectory));
auto rdsPos = devicePathView.rfind(renderDeviceSuffix);
if (rdsPos == std::string::npos) {
continue;
}
if (rdsPos < devicePathView.size() - strlen(renderDeviceSuffix)) {
continue;
}
// at least 'pci-0000:00:00.0' -> 16
if (rdsPos < 16 || devicePathView[rdsPos - 13] != '-') {
continue;
}
std::string pciPath(devicePathView.substr(rdsPos - 12, 12));
if (!osPciPath.empty()) {
if (osPciPath.compare(pciPath) != 0) {
// if osPciPath is non-empty, then interest is only in discovering device having same bdf as ocPciPath. Skip all other devices.
continue;
}
}
if (debugManager.flags.FilterBdfPath.get() != "unk") {
if (devicePathView.find(debugManager.flags.FilterBdfPath.get().c_str()) == std::string::npos) {
continue;
}
}
int fileDescriptor = SysCalls::open(file->c_str(), O_RDWR | O_CLOEXEC);
appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.c_str(), file->c_str());
if (!hwDeviceIds.empty() && hwDeviceIds.size() == numRootDevices) {
break;
}
}
if (hwDeviceIds.empty()) {
return hwDeviceIds;
}
} while (hwDeviceIds.size() < numRootDevices);
return hwDeviceIds;
}
std::string Drm::getDrmVersion(int fileDescriptor) {
DrmVersion version = {};
char name[5] = {};
version.name = name;
version.nameLen = 5;
auto requestValue = getIoctlRequestValue(DrmIoctl::version, nullptr);
int ret = SysCalls::ioctl(fileDescriptor, requestValue, &version);
if (ret) {
return {};
}
name[4] = '\0';
return std::string(name);
}
template <typename DataType>
std::vector<DataType> Drm::query(uint32_t queryId, uint32_t queryItemFlags) {
Query query{};
QueryItem queryItem{};
queryItem.queryId = queryId;
queryItem.length = 0; // query length first
queryItem.flags = queryItemFlags;
query.itemsPtr = reinterpret_cast<uint64_t>(&queryItem);
query.numItems = 1;
auto ret = ioctlHelper->ioctl(DrmIoctl::query, &query);
if (ret != 0 || queryItem.length <= 0) {
return {};
}
auto data = std::vector<DataType>(Math::divideAndRoundUp(queryItem.length, sizeof(DataType)), 0);
queryItem.dataPtr = castToUint64(data.data());
ret = ioctlHelper->ioctl(DrmIoctl::query, &query);
if (ret != 0 || queryItem.length <= 0) {
return {};
}
return data;
}
void Drm::printIoctlStatistics() {
if (!debugManager.flags.PrintKmdTimes.get()) {
return;
}
printf("\n--- Ioctls statistics ---\n");
printf("%41s %15s %10s %20s %20s %20s", "Request", "Total time(ns)", "Count", "Avg time per ioctl", "Min", "Max\n");
for (const auto &ioctlData : this->ioctlStatistics) {
printf("%41s %15llu %10lu %20f %20lld %20lld\n",
ioctlHelper->getIoctlString(ioctlData.first).c_str(),
ioctlData.second.totalTime,
static_cast<unsigned long>(ioctlData.second.count),
ioctlData.second.totalTime / static_cast<double>(ioctlData.second.count),
ioctlData.second.minTime,
ioctlData.second.maxTime);
}
printf("\n");
}
bool Drm::createVirtualMemoryAddressSpace(uint32_t vmCount) {
for (auto i = 0u; i < vmCount; i++) {
uint32_t id = i;
if (0 != createDrmVirtualMemory(id)) {
return false;
}
virtualMemoryIds.push_back(id);
}
return true;
}
void Drm::destroyVirtualMemoryAddressSpace() {
for (auto id : virtualMemoryIds) {
destroyDrmVirtualMemory(id);
}
virtualMemoryIds.clear();
}
uint32_t Drm::getVirtualMemoryAddressSpace(uint32_t vmId) const {
if (vmId < virtualMemoryIds.size()) {
return virtualMemoryIds[vmId];
}
return 0;
}
void Drm::setNewResourceBoundToVM(BufferObject *bo, uint32_t vmHandleId) {
if (!this->rootDeviceEnvironment.getProductHelper().isTlbFlushRequired()) {
return;
}
const auto &engines = this->rootDeviceEnvironment.executionEnvironment.memoryManager->getRegisteredEngines(bo->getRootDeviceIndex());
for (const auto &engine : engines) {
if (engine.osContext->getDeviceBitfield().test(vmHandleId)) {
auto osContextLinux = static_cast<OsContextLinux *>(engine.osContext);
osContextLinux->setNewResourceBound();
}
}
}
PhysicalDevicePciBusInfo Drm::getPciBusInfo() const {
PhysicalDevicePciBusInfo pciBusInfo(PhysicalDevicePciBusInfo::invalidValue, PhysicalDevicePciBusInfo::invalidValue, PhysicalDevicePciBusInfo::invalidValue, PhysicalDevicePciBusInfo::invalidValue);
if (adapterBDF.Data != std::numeric_limits<uint32_t>::max()) {
pciBusInfo.pciDomain = this->pciDomain;
pciBusInfo.pciBus = adapterBDF.Bus;
pciBusInfo.pciDevice = adapterBDF.Device;
pciBusInfo.pciFunction = adapterBDF.Function;
}
return pciBusInfo;
}
void Drm::cleanup() {
destroyVirtualMemoryAddressSpace();
}
Drm::~Drm() {
this->printIoctlStatistics();
}
int Drm::queryAdapterBDF() {
constexpr int pciBusInfoTokensNum = 4;
uint16_t domain = -1;
uint8_t bus = -1, device = -1, function = -1;
if (NEO::parseBdfString(hwDeviceId->getPciPath(), domain, bus, device, function) != pciBusInfoTokensNum) {
adapterBDF.Data = std::numeric_limits<uint32_t>::max();
return 1;
}
setPciDomain(domain);
adapterBDF.Bus = bus;
adapterBDF.Function = function;
adapterBDF.Device = device;
return 0;
}
void Drm::setGmmInputArgs(void *args) {
auto gmmInArgs = reinterpret_cast<GMM_INIT_IN_ARGS *>(args);
#if defined(__linux__)
gmmInArgs->FileDescriptor = adapterBDF.Data;
#endif
gmmInArgs->ClientType = GMM_CLIENT::GMM_OCL_VISTA;
}
const std::vector<int> &Drm::getSliceMappings(uint32_t deviceIndex) {
return topologyMap[deviceIndex].sliceIndices;
}
int Drm::waitHandle(uint32_t waitHandle, int64_t timeout) {
UNRECOVERABLE_IF(isVmBindAvailable());
GemWait wait{};
wait.boHandle = waitHandle;
wait.timeoutNs = timeout;
StackVec<std::unique_lock<NEO::CommandStreamReceiver::MutexType>, 1> locks{};
if (this->rootDeviceEnvironment.executionEnvironment.memoryManager) {
const auto &mulitEngines = this->rootDeviceEnvironment.executionEnvironment.memoryManager->getRegisteredEngines();
for (const auto &engines : mulitEngines) {
for (const auto &engine : engines) {
if (engine.osContext->isDirectSubmissionLightActive()) {
locks.push_back(engine.commandStreamReceiver->obtainUniqueOwnership());
engine.commandStreamReceiver->stopDirectSubmission(false, false);
}
}
}
}
int ret = ioctlHelper->ioctl(DrmIoctl::gemWait, &wait);
if (ret != 0) {
int err = errno;
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(I915_GEM_WAIT) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
}
return ret;
}
int Drm::getTimestampFrequency(int &frequency) {
frequency = 0;
return getParamIoctl(DrmParam::paramCsTimestampFrequency, &frequency);
}
int Drm::getOaTimestampFrequency(int &frequency) {
frequency = 0;
return getParamIoctl(DrmParam::paramOATimestampFrequency, &frequency);
}
bool Drm::queryEngineInfo() {
UNRECOVERABLE_IF(!memoryInfoQueried);
UNRECOVERABLE_IF(engineInfoQueried);
engineInfoQueried = true;
return Drm::queryEngineInfo(false);
}
bool Drm::sysmanQueryEngineInfo() {
return Drm::queryEngineInfo(true);
}
bool Drm::isDebugAttachAvailable() {
int enableEuDebug = getEuDebugSysFsEnable();
return (enableEuDebug == 1) && ioctlHelper->isDebugAttachAvailable();
}
int Drm::getEuDebugSysFsEnable() {
return ioctlHelper->getEuDebugSysFsEnable();
}
int getMaxGpuFrequencyOfDevice(Drm &drm, std::string &sysFsPciPath, int &maxGpuFrequency) {
maxGpuFrequency = 0;
std::string clockSysFsPath = sysFsPciPath + drm.getIoctlHelper()->getFileForMaxGpuFrequency();
std::ifstream ifs(clockSysFsPath.c_str(), std::ifstream::in);
if (ifs.fail()) {
return -1;
}
ifs >> maxGpuFrequency;
ifs.close();
return 0;
}
int getMaxGpuFrequencyOfSubDevice(Drm &drm, std::string &sysFsPciPath, int subDeviceId, int &maxGpuFrequency) {
maxGpuFrequency = 0;
std::string clockSysFsPath = sysFsPciPath + drm.getIoctlHelper()->getFileForMaxGpuFrequencyOfSubDevice(subDeviceId);
std::ifstream ifs(clockSysFsPath.c_str(), std::ifstream::in);
if (ifs.fail()) {
return -1;
}
ifs >> maxGpuFrequency;
ifs.close();
return 0;
}
int Drm::getMaxGpuFrequency(HardwareInfo &hwInfo, int &maxGpuFrequency) {
int ret = 0;
std::string sysFsPciPath = getSysFsPciPath();
auto tileCount = hwInfo.gtSystemInfo.MultiTileArchInfo.TileCount;
if (hwInfo.gtSystemInfo.MultiTileArchInfo.IsValid && tileCount > 0) {
for (auto tileId = 0; tileId < tileCount; tileId++) {
int maxGpuFreqOfSubDevice = 0;
ret |= getMaxGpuFrequencyOfSubDevice(*this, sysFsPciPath, tileId, maxGpuFreqOfSubDevice);
maxGpuFrequency = std::max(maxGpuFrequency, maxGpuFreqOfSubDevice);
}
if (ret == 0) {
return 0;
}
}
return getMaxGpuFrequencyOfDevice(*this, sysFsPciPath, maxGpuFrequency);
}
bool Drm::getDeviceMemoryMaxClockRateInMhz(uint32_t tileId, uint32_t &clkRate) {
const std::string relativefilePath = ioctlHelper->getFileForMaxMemoryFrequencyOfSubDevice(tileId);
std::string readString(64, '\0');
errno = 0;
if (readSysFsAsString(relativefilePath, readString) == false) {
return false;
}
char *endPtr = nullptr;
uint32_t retClkRate = static_cast<uint32_t>(std::strtoul(readString.data(), &endPtr, 10));
if ((endPtr == readString.data()) || (errno != 0)) {
return false;
}
clkRate = retClkRate;
return true;
}
bool Drm::getDeviceMemoryPhysicalSizeInBytes(uint32_t tileId, uint64_t &physicalSize) {
if (memoryInfo == nullptr || memoryInfo->getLocalMemoryRegions().size() == 0U) {
physicalSize = 0U;
return false;
}
physicalSize = memoryInfo->getLocalMemoryRegionSize(tileId);
return true;
}
bool Drm::useVMBindImmediate() const {
bool useBindImmediate = isDirectSubmissionActive() || hasPageFaultSupport() || ioctlHelper->isImmediateVmBindRequired();
if (debugManager.flags.EnableImmediateVmBindExt.get() != -1) {
useBindImmediate = debugManager.flags.EnableImmediateVmBindExt.get();
}
return useBindImmediate;
}
void Drm::setupSystemInfo(HardwareInfo *hwInfo, SystemInfo *sysInfo) {
GT_SYSTEM_INFO *gtSysInfo = &hwInfo->gtSystemInfo;
gtSysInfo->MaxEuPerSubSlice = sysInfo->getMaxEuPerDualSubSlice();
gtSysInfo->MemoryType = sysInfo->getMemoryType();
gtSysInfo->MaxSlicesSupported = sysInfo->getMaxSlicesSupported();
gtSysInfo->MaxSubSlicesSupported = sysInfo->getMaxDualSubSlicesSupported();
gtSysInfo->MaxDualSubSlicesSupported = sysInfo->getMaxDualSubSlicesSupported();
gtSysInfo->CsrSizeInMb = sysInfo->getCsrSizeInMb();
gtSysInfo->SLMSizeInKb = sysInfo->getSlmSizePerDss();
}
void Drm::setupCacheInfo(const HardwareInfo &hwInfo) {
auto &productHelper = rootDeviceEnvironment.getHelper<ProductHelper>();
if (debugManager.flags.ForceStaticL2ClosReservation.get()) {
if (debugManager.flags.L2ClosNumCacheWays.get() == -1) {
debugManager.flags.L2ClosNumCacheWays.set(2U);
}
}
auto getL2CacheReservationLimits{[&productHelper]() {
CacheReservationParameters out{};
if (productHelper.getNumCacheRegions() == 0) {
return out;
}
if (auto numCacheWays{debugManager.flags.L2ClosNumCacheWays.get()}; numCacheWays != -1) {
out.maxSize = 1U;
out.maxNumRegions = 1U;
out.maxNumWays = static_cast<uint16_t>(numCacheWays);
return out;
}
return out;
}};
auto getL3CacheReservationLimits{[&hwInfo, &productHelper]() {
CacheReservationParameters out{};
if (debugManager.flags.ClosEnabled.get() == 0 || productHelper.getNumCacheRegions() == 0) {
return out;
}
constexpr uint16_t totalMaxNumWays = 32U;
constexpr uint16_t globalReservationLimit = 16U;
constexpr uint16_t clientReservationLimit = 8U;
const size_t totalCacheSize = hwInfo.gtSystemInfo.L3CacheSizeInKb * MemoryConstants::kiloByte;
out.maxNumWays = std::min(globalReservationLimit, clientReservationLimit);
out.maxSize = (totalCacheSize * out.maxNumWays) / totalMaxNumWays;
out.maxNumRegions = productHelper.getNumCacheRegions() - 1;
return out;
}};
this->cacheInfo.reset(new CacheInfo(*ioctlHelper, getL2CacheReservationLimits(), getL3CacheReservationLimits()));
if (debugManager.flags.ForceStaticL2ClosReservation.get()) {
[[maybe_unused]] bool isReserved{this->cacheInfo->getCacheRegion(getL2CacheReservationLimits().maxSize, CacheRegion::region3)};
DEBUG_BREAK_IF(!isReserved);
}
}
void Drm::getPrelimVersion(std::string &prelimVersion) {
std::string sysFsPciPath = getSysFsPciPath();
std::string prelimVersionPath = sysFsPciPath + "/prelim_uapi_version";
std::ifstream ifs(prelimVersionPath.c_str(), std::ifstream::in);
if (ifs.fail()) {
prelimVersion = "";
} else {
ifs >> prelimVersion;
}
ifs.close();
}
int Drm::waitUserFence(uint32_t ctxId, uint64_t address, uint64_t value, ValueWidth dataWidth, int64_t timeout, uint16_t flags, bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) {
return ioctlHelper->waitUserFence(ctxId, address, value, static_cast<uint32_t>(dataWidth), timeout, flags, userInterrupt, externalInterruptId, allocForInterruptWait);
}
bool Drm::querySystemInfo() {
if (systemInfoQueried) {
return this->systemInfo != nullptr;
}
systemInfoQueried = true;
auto request = ioctlHelper->getDrmParamValue(DrmParam::queryHwconfigTable);
auto deviceBlobQuery = this->query<uint32_t>(request, 0);
if (deviceBlobQuery.empty()) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stdout, "%s", "INFO: System Info query failed!\n");
return false;
}
this->systemInfo.reset(new SystemInfo(deviceBlobQuery));
return true;
}
std::vector<uint64_t> Drm::getMemoryRegions() {
auto request = ioctlHelper->getDrmParamValue(DrmParam::queryMemoryRegions);
return this->query<uint64_t>(request, 0);
}
bool Drm::queryMemoryInfo() {
UNRECOVERABLE_IF(memoryInfoQueried);
this->memoryInfo = ioctlHelper->createMemoryInfo();
memoryInfoQueried = true;
return this->memoryInfo != nullptr;
}
bool Drm::queryEngineInfo(bool isSysmanEnabled) {
this->engineInfo = ioctlHelper->createEngineInfo(isSysmanEnabled);
if (this->engineInfo && (this->engineInfo->hasEngines() == false)) {
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Engine info size is equal to 0.\n");
}
return this->engineInfo != nullptr;
}
bool Drm::completionFenceSupport() {
std::call_once(checkCompletionFenceOnce, [this]() {
const bool vmBindAvailable = isVmBindAvailable();
bool support = ioctlHelper->completionFenceExtensionSupported(vmBindAvailable);
int32_t overrideCompletionFence = debugManager.flags.EnableDrmCompletionFence.get();
if (overrideCompletionFence != -1) {
support = !!overrideCompletionFence;
}
completionFenceSupported = support;
if (debugManager.flags.PrintCompletionFenceUsage.get()) {
std::cout << "Completion fence supported: " << completionFenceSupported << std::endl;
}
});
return completionFenceSupported;
}
void Drm::setupIoctlHelper(const PRODUCT_FAMILY productFamily) {
if (!this->ioctlHelper) {
auto drmVersion = Drm::getDrmVersion(getFileDescriptor());
auto productSpecificIoctlHelperCreator = ioctlHelperFactory[productFamily];
if (productSpecificIoctlHelperCreator && !debugManager.flags.IgnoreProductSpecificIoctlHelper.get()) {
this->ioctlHelper = productSpecificIoctlHelperCreator.value()(*this);
} else if ("xe" == drmVersion) {
this->ioctlHelper = IoctlHelperXe::create(*this);
} else {
std::string prelimVersion = "";
getPrelimVersion(prelimVersion);
this->ioctlHelper = IoctlHelper::getI915Helper(productFamily, prelimVersion, *this);
}
this->ioctlHelper->initialize();
}
}
bool Drm::queryTopology(const HardwareInfo &hwInfo, DrmQueryTopologyData &topologyData) {
UNRECOVERABLE_IF(!systemInfoQueried);
UNRECOVERABLE_IF(!engineInfoQueried);
UNRECOVERABLE_IF(topologyQueried);
topologyQueried = true;
auto result = this->ioctlHelper->getTopologyDataAndMap(hwInfo, topologyData, topologyMap);
return result;
}
void Drm::queryPageFaultSupport() {
const auto &productHelper = this->getRootDeviceEnvironment().getHelper<ProductHelper>();
if (!productHelper.isPageFaultSupported()) {
return;
}
pageFaultSupported = this->ioctlHelper->isPageFaultSupported();
}
bool Drm::hasPageFaultSupport() const {
if (debugManager.flags.EnableRecoverablePageFaults.get() != -1) {
return !!debugManager.flags.EnableRecoverablePageFaults.get();
}
return pageFaultSupported;
}
bool Drm::hasKmdMigrationSupport() const {
const auto &productHelper = this->getRootDeviceEnvironment().getHelper<ProductHelper>();
auto kmdMigrationSupported = hasPageFaultSupport() && productHelper.isKmdMigrationSupported();
if (debugManager.flags.UseKmdMigration.get() != -1) {
return !!debugManager.flags.UseKmdMigration.get();
}
return kmdMigrationSupported;
}
void Drm::configureScratchPagePolicy() {
if (debugManager.flags.DisableScratchPages.get() != -1) {
disableScratch = !!debugManager.flags.DisableScratchPages.get();
return;
}
const auto &productHelper = this->getRootDeviceEnvironment().getHelper<ProductHelper>();
if (rootDeviceEnvironment.executionEnvironment.isDebuggingEnabled()) {
disableScratch = productHelper.isDisableScratchPagesRequiredForDebugger();
} else {
disableScratch = productHelper.isDisableScratchPagesSupported();
}
}
void Drm::configureGpuFaultCheckThreshold() {
if (debugManager.flags.GpuFaultCheckThreshold.get() != -1) {
gpuFaultCheckThreshold = debugManager.flags.GpuFaultCheckThreshold.get();
}
}
unsigned int Drm::bindDrmContext(uint32_t drmContextId, uint32_t deviceIndex, aub_stream::EngineType engineType) {
auto engineInfo = this->engineInfo.get();
auto retVal = static_cast<unsigned int>(ioctlHelper->getDrmParamValue(DrmEngineMapper::engineNodeMap(engineType)));
if (!engineInfo) {
return retVal;
}
auto engine = engineInfo->getEngineInstance(deviceIndex, engineType);
if (!engine) {
return retVal;
}
bool useVirtualEnginesForCcs = true;
if (debugManager.flags.UseDrmVirtualEnginesForCcs.get() != -1) {
useVirtualEnginesForCcs = !!debugManager.flags.UseDrmVirtualEnginesForCcs.get();
}
auto numberOfCCS = rootDeviceEnvironment.getHardwareInfo()->gtSystemInfo.CCSInfo.NumberOfCCSEnabled;
constexpr uint32_t maxEngines = 9u;
bool useVirtualEnginesForBcs = EngineHelpers::isBcsVirtualEngineEnabled(engineType);
auto numberOfBCS = rootDeviceEnvironment.getHardwareInfo()->featureTable.ftrBcsInfo.count();
if (debugManager.flags.LimitEngineCountForVirtualBcs.get() != -1) {
numberOfBCS = debugManager.flags.LimitEngineCountForVirtualBcs.get();
}
if (debugManager.flags.LimitEngineCountForVirtualCcs.get() != -1) {
numberOfCCS = debugManager.flags.LimitEngineCountForVirtualCcs.get();
}
uint32_t numEnginesInContext = 1;
ContextParamEngines<> contextEngines{};
ContextEnginesLoadBalance<maxEngines> balancer{};
ioctlHelper->insertEngineToContextParams(contextEngines, 0u, engine, deviceIndex, false);
bool setupVirtualEngines = false;
unsigned int engineCount = static_cast<unsigned int>(numberOfCCS);
if (useVirtualEnginesForCcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCompute) && numberOfCCS > 1u) {
numEnginesInContext = numberOfCCS + 1;
balancer.numSiblings = numberOfCCS;
setupVirtualEngines = true;
}
bool includeMainCopyEngineInGroup = false;
if (useVirtualEnginesForBcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCopy) && numberOfBCS > 1u) {
numEnginesInContext = static_cast<uint32_t>(numberOfBCS) + 1;
balancer.numSiblings = numberOfBCS;
setupVirtualEngines = true;
engineCount = static_cast<unsigned int>(rootDeviceEnvironment.getHardwareInfo()->featureTable.ftrBcsInfo.size());
if (EngineHelpers::getBcsIndex(engineType) == 0u) {
includeMainCopyEngineInGroup = true;
} else {
engineCount--;
balancer.numSiblings = numberOfBCS - 1;
numEnginesInContext = static_cast<uint32_t>(numberOfBCS);
}
}
if (setupVirtualEngines) {
balancer.base.name = ioctlHelper->getDrmParamValue(DrmParam::contextEnginesExtLoadBalance);
contextEngines.extensions = castToUint64(&balancer);
ioctlHelper->insertEngineToContextParams(contextEngines, 0u, nullptr, deviceIndex, true);
for (auto engineIndex = 0u; engineIndex < engineCount; engineIndex++) {
if (useVirtualEnginesForBcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCopy)) {
auto mappedBcsEngineType = static_cast<aub_stream::EngineType>(EngineHelpers::mapBcsIndexToEngineType(engineIndex, includeMainCopyEngineInGroup));
bool isBcsEnabled = rootDeviceEnvironment.getHardwareInfo()->featureTable.ftrBcsInfo.test(EngineHelpers::getBcsIndex(mappedBcsEngineType));
if (!isBcsEnabled) {
continue;
}
engine = engineInfo->getEngineInstance(deviceIndex, mappedBcsEngineType);
}
UNRECOVERABLE_IF(!engine);
if (useVirtualEnginesForCcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCompute)) {
engine = engineInfo->getEngineInstance(deviceIndex, static_cast<aub_stream::EngineType>(EngineHelpers::mapCcsIndexToEngineType(engineIndex)));
}
UNRECOVERABLE_IF(!engine);
balancer.engines[engineIndex] = {engine->engineClass, engine->engineInstance};
ioctlHelper->insertEngineToContextParams(contextEngines, engineIndex, engine, deviceIndex, true);
}
}
GemContextParam param{};
param.contextId = drmContextId;
param.size = static_cast<uint32_t>(ptrDiff(contextEngines.enginesData, &contextEngines) + sizeof(EngineClassInstance) * numEnginesInContext);
param.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamEngines);
param.value = castToUint64(&contextEngines);
auto ioctlValue = ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &param);
UNRECOVERABLE_IF(ioctlValue != 0);
retVal = static_cast<unsigned int>(ioctlHelper->getDrmParamValue(DrmParam::execDefault));
return retVal;
}
void Drm::waitForBind(uint32_t vmHandleId) {
auto fenceAddressAndValToWait = getFenceAddressAndValToWait(vmHandleId, false);
const auto fenceAddressToWait = fenceAddressAndValToWait.first;
const auto fenceValToWait = fenceAddressAndValToWait.second;
if (fenceAddressToWait != 0u) {
waitUserFence(0u, fenceAddressToWait, fenceValToWait, ValueWidth::u64, -1, ioctlHelper->getWaitUserFenceSoftFlag(), false, NEO::InterruptId::notUsed, nullptr);
}
}
std::pair<uint64_t, uint64_t> Drm::getFenceAddressAndValToWait(uint32_t vmHandleId, bool isLocked) {
std::pair<uint64_t, uint64_t> fenceAddressAndValToWait = std::make_pair(0, 0);
std::unique_lock<std::mutex> lock;
if (!isLocked) {
lock = this->lockBindFenceMutex();
}
if (!(*ioctlHelper->getPagingFenceAddress(vmHandleId, nullptr) >= fenceVal[vmHandleId])) {
auto fenceAddress = castToUint64(ioctlHelper->getPagingFenceAddress(vmHandleId, nullptr));
auto fenceValue = this->fenceVal[vmHandleId];
fenceAddressAndValToWait = std::make_pair(fenceAddress, fenceValue);
}
if (!isLocked) {
lock.unlock();
}
return fenceAddressAndValToWait;
}
bool Drm::isSetPairAvailable() {
if (debugManager.flags.EnableSetPair.get() == 1) {
std::call_once(checkSetPairOnce, [this]() {
int ret = ioctlHelper->isSetPairAvailable();
setPairAvailable = ret;
});
}
return setPairAvailable;
}
bool Drm::isChunkingAvailable() {
if (debugManager.flags.EnableBOChunking.get() != 0) {
std::call_once(checkChunkingOnce, [this]() {
int ret = ioctlHelper->isChunkingAvailable();
if (ret) {
if (debugManager.flags.EnableBOChunking.get() == -1) {
chunkingMode = chunkingModeDevice;
} else {
chunkingMode = debugManager.flags.EnableBOChunking.get();
if (!(hasKmdMigrationSupport())) {
chunkingMode &= (~(chunkingModeShared));
}
}
}
if (chunkingMode > 0) {
chunkingAvailable = true;
}
if (debugManager.flags.MinimalAllocationSizeForChunking.get() != -1) {
minimalChunkingSize = debugManager.flags.MinimalAllocationSizeForChunking.get();
}
printDebugString(debugManager.flags.PrintBOChunkingLogs.get(), stdout,
"Chunking available: %d; enabled for: shared allocations %d, device allocations %d; minimalChunkingSize: %zd\n",
chunkingAvailable,
(chunkingMode & chunkingModeShared),
(chunkingMode & chunkingModeDevice),
minimalChunkingSize);
});
}
return chunkingAvailable;
}
bool Drm::isVmBindAvailable() {
std::call_once(checkBindOnce, [this]() {
bindAvailable = ioctlHelper->isVmBindAvailable();
Drm::overrideBindSupport(bindAvailable);
queryAndSetVmBindPatIndexProgrammingSupport();
});
return bindAvailable;
}
uint64_t Drm::getPatIndex(Gmm *gmm, AllocationType allocationType, CacheRegion cacheRegion, CachePolicy cachePolicy, bool closEnabled, bool isSystemMemory) const {
if ((debugManager.flags.OverridePatIndexForSystemMemory.get() != -1) && isSystemMemory) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForSystemMemory.get());
}
if ((debugManager.flags.OverridePatIndexForDeviceMemory.get() != -1) && !isSystemMemory) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForDeviceMemory.get());
}
if (debugManager.flags.OverridePatIndex.get() != -1) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndex.get());
}
auto &productHelper = rootDeviceEnvironment.getProductHelper();
GMM_RESOURCE_USAGE_TYPE usageType = CacheSettingsHelper::getGmmUsageType(allocationType, false, productHelper, getHardwareInfo());
auto isUncachedType = CacheSettingsHelper::isUncachedType(usageType);
if (isUncachedType && debugManager.flags.OverridePatIndexForUncachedTypes.get() != -1) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForUncachedTypes.get());
}
if (!isUncachedType && debugManager.flags.OverridePatIndexForCachedTypes.get() != -1) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForCachedTypes.get());
}
if (!this->vmBindPatIndexProgrammingSupported) {
return CommonConstants::unsupportedPatIndex;
}
GMM_RESOURCE_INFO *resourceInfo = nullptr;
bool cachable = !CacheSettingsHelper::isUncachedType(usageType);
bool compressed = false;
if (gmm) {
resourceInfo = gmm->gmmResourceInfo->peekGmmResourceInfo();
usageType = gmm->resourceParams.Usage;
compressed = gmm->isCompressionEnabled();
cachable = gmm->gmmResourceInfo->getResourceFlags()->Info.Cacheable;
}
uint64_t patIndex = rootDeviceEnvironment.getGmmClientContext()->cachePolicyGetPATIndex(resourceInfo, usageType, compressed, cachable);
patIndex = productHelper.overridePatIndex(isUncachedType, patIndex, allocationType);
UNRECOVERABLE_IF(patIndex == static_cast<uint64_t>(GMM_PAT_ERROR));
if (debugManager.flags.ClosEnabled.get() != -1) {
closEnabled = !!debugManager.flags.ClosEnabled.get();
}
if (closEnabled) {
patIndex = productHelper.getPatIndex(cacheRegion, cachePolicy);
}
return patIndex;
}
void programUserFence(Drm *drm, OsContext *osContext, BufferObject *bo, VmBindExtUserFenceT &vmBindExtUserFence, uint32_t vmHandleId, uint64_t nextExtension) {
auto ioctlHelper = drm->getIoctlHelper();
uint64_t address = 0;
uint64_t value = 0;
if (drm->isPerContextVMRequired()) {
auto osContextLinux = static_cast<OsContextLinux *>(osContext);
address = castToUint64(ioctlHelper->getPagingFenceAddress(vmHandleId, osContextLinux));
value = osContextLinux->getNextFenceVal(vmHandleId);
} else {
address = castToUint64(ioctlHelper->getPagingFenceAddress(vmHandleId, nullptr));
value = drm->getNextFenceVal(vmHandleId);
}
ioctlHelper->fillVmBindExtUserFence(vmBindExtUserFence, address, value, nextExtension);
}
int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleId, BufferObject *bo, bool bind, const bool forcePagingFence) {
auto vmId = drm->getVirtualMemoryAddressSpace(vmHandleId);
auto ioctlHelper = drm->getIoctlHelper();
uint64_t flags = 0u;
if (drm->isPerContextVMRequired()) {
auto osContextLinux = static_cast<const OsContextLinux *>(osContext);
UNRECOVERABLE_IF(osContextLinux->getDrmVmIds().size() <= vmHandleId);
vmId = osContextLinux->getDrmVmIds()[vmHandleId];
}
// Use only when debugger is disabled
const bool guaranteePagingFence = forcePagingFence && !drm->getRootDeviceEnvironment().executionEnvironment.isDebuggingEnabled();
std::unique_ptr<uint8_t[]> extensions;
if (bind) {
bool allowUUIDsForDebug = !osContext->isInternalEngine() && !EngineHelpers::isBcs(osContext->getEngineType());
if (bo->getBindExtHandles().size() > 0 && allowUUIDsForDebug) {
extensions = ioctlHelper->prepareVmBindExt(bo->getBindExtHandles(), bo->getRegisteredBindHandleCookie());
}
bool bindCapture = bo->isMarkedForCapture();
bool bindImmediate = bo->isImmediateBindingRequired();
bool bindMakeResident = false;
bool readOnlyResource = bo->isReadOnlyGpuResource();
if (drm->useVMBindImmediate() || guaranteePagingFence) {
bindMakeResident = bo->isExplicitResidencyRequired();
bindImmediate = true;
}
bool bindLock = bo->isExplicitLockedMemoryRequired();
flags |= ioctlHelper->getFlagsForVmBind(bindCapture, bindImmediate, bindMakeResident, bindLock, readOnlyResource);
}
auto &bindAddresses = bo->getColourAddresses();
auto bindIterations = bindAddresses.size();
if (bindIterations == 0) {
bindIterations = 1;
}
int ret = 0;
for (size_t i = 0; i < bindIterations; i++) {
VmBindParams vmBind{};
vmBind.vmId = static_cast<uint32_t>(vmId);
vmBind.flags = flags;
vmBind.handle = bo->peekHandle();
vmBind.length = bo->peekSize();
vmBind.offset = 0;
vmBind.start = bo->peekAddress();
vmBind.userptr = bo->getUserptr();
vmBind.sharedSystemUsmEnabled = drm->isSharedSystemAllocEnabled();
vmBind.sharedSystemUsmBind = false;
if (bo->getColourWithBind()) {
vmBind.length = bo->getColourChunk();
vmBind.offset = bo->getColourChunk() * i;
vmBind.start = bindAddresses[i];
}
VmBindExtSetPatT vmBindExtSetPat{};
if (drm->isVmBindPatIndexProgrammingSupported()) {
UNRECOVERABLE_IF(bo->peekPatIndex() == CommonConstants::unsupportedPatIndex);
if (ioctlHelper->isVmBindPatIndexExtSupported()) {
ioctlHelper->fillVmBindExtSetPat(vmBindExtSetPat, bo->peekPatIndex(), castToUint64(extensions.get()));
vmBind.extensions = castToUint64(vmBindExtSetPat);
} else {
vmBind.extensions = castToUint64(extensions.get());
}
vmBind.patIndex = bo->peekPatIndex();
} else {
vmBind.extensions = castToUint64(extensions.get());
}
std::unique_lock<std::mutex> lock;
VmBindExtUserFenceT vmBindExtUserFence{};
bool incrementFenceValue = false;
if ((ioctlHelper->isWaitBeforeBindRequired(bind) && drm->useVMBindImmediate()) || guaranteePagingFence) {
lock = drm->lockBindFenceMutex();
auto nextExtension = vmBind.extensions;
incrementFenceValue = true;
programUserFence(drm, osContext, bo, vmBindExtUserFence, vmHandleId, nextExtension);
ioctlHelper->setVmBindUserFence(vmBind, vmBindExtUserFence);
}
if (bind) {
ret = ioctlHelper->vmBind(vmBind);
if (ret) {
break;
}
drm->setNewResourceBoundToVM(bo, vmHandleId);
} else {
vmBind.handle = 0u;
ret = ioctlHelper->vmUnbind(vmBind);
if (ret) {
break;
}
}
if (incrementFenceValue) {
auto osContextLinux = static_cast<OsContextLinux *>(osContext);
std::pair<uint64_t, uint64_t> fenceAddressAndValToWait = osContextLinux->getFenceAddressAndValToWait(vmHandleId, true);
if (drm->isPerContextVMRequired()) {
osContextLinux->incFenceVal(vmHandleId);
} else {
drm->incFenceVal(vmHandleId);
}
lock.unlock();
const auto fenceAddressToWait = fenceAddressAndValToWait.first;
const auto fenceValToWait = fenceAddressAndValToWait.second;
if (fenceAddressToWait != 0u) {
bool waitOnUserFenceAfterBindAndUnbind = false;
if (debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get() != -1) {
waitOnUserFenceAfterBindAndUnbind = !!debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get();
}
if ((ioctlHelper->isWaitBeforeBindRequired(bind) && waitOnUserFenceAfterBindAndUnbind && drm->useVMBindImmediate()) || guaranteePagingFence) {
drm->waitUserFence(0u, fenceAddressToWait, fenceValToWait, Drm::ValueWidth::u64, -1, ioctlHelper->getWaitUserFenceSoftFlag(), false, NEO::InterruptId::notUsed, nullptr);
}
}
}
}
return ret;
}
int Drm::bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo, const bool forcePagingFence) {
auto ret = changeBufferObjectBinding(this, osContext, vmHandleId, bo, true, forcePagingFence);
if (ret != 0) {
static_cast<DrmMemoryOperationsHandlerBind *>(this->rootDeviceEnvironment.memoryOperationsInterface.get())->evictUnusedAllocations(false, false);
ret = changeBufferObjectBinding(this, osContext, vmHandleId, bo, true, forcePagingFence);
}
return ret;
}
int Drm::unbindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) {
return changeBufferObjectBinding(this, osContext, vmHandleId, bo, false, false);
}
int Drm::createDrmVirtualMemory(uint32_t &drmVmId) {
GemVmControl ctl{};
std::optional<MemoryClassInstance> regionInstanceClass;
uint32_t memoryBank = 1 << drmVmId;
auto hwInfo = this->getRootDeviceEnvironment().getHardwareInfo();
auto memInfo = this->getMemoryInfo();
if (debugManager.flags.UseTileMemoryBankInVirtualMemoryCreation.get() != 0) {
if (memInfo && rootDeviceEnvironment.getHelper<GfxCoreHelper>().getEnableLocalMemory(*hwInfo)) {
regionInstanceClass = memInfo->getMemoryRegionClassAndInstance(memoryBank, *this->rootDeviceEnvironment.getHardwareInfo());
}
}
auto vmControlExtRegion = ioctlHelper->createVmControlExtRegion(regionInstanceClass);
if (vmControlExtRegion) {
ctl.extensions = castToUint64(vmControlExtRegion.get());
}
bool useVmBind = isVmBindAvailable();
bool enablePageFault = hasPageFaultSupport() && useVmBind;
ctl.flags = ioctlHelper->getFlagsForVmCreate(checkToDisableScratchPage(), enablePageFault, useVmBind);
auto ret = ioctlHelper->ioctl(DrmIoctl::gemVmCreate, &ctl);
if (ret == 0) {
drmVmId = ctl.vmId;
if (isSharedSystemAllocEnabled()) {
VmBindParams vmBind{};
vmBind.vmId = static_cast<uint32_t>(ctl.vmId);
vmBind.flags = DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR;
vmBind.length = (0x1ull << ((NEO::CpuInfo::getInstance().getVirtualAddressSize()) - 1));
vmBind.sharedSystemUsmEnabled = true;
vmBind.sharedSystemUsmBind = true;
VmBindExtUserFenceT vmBindExtUserFence{};
ioctlHelper->fillVmBindExtUserFence(vmBindExtUserFence,
castToUint64(ioctlHelper->getPagingFenceAddress(0, nullptr)),
getNextFenceVal(0),
vmBind.extensions);
ioctlHelper->setVmBindUserFence(vmBind, vmBindExtUserFence);
if (ioctlHelper->vmBind(vmBind)) {
setSharedSystemAllocEnable(false);
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr,
"INFO: Shared System USM capability not detected\n");
}
}
if (ctl.vmId == 0) {
// 0 is reserved for invalid/unassigned ppgtt
return -1;
}
} else {
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr,
"INFO: Cannot create Virtual Memory at memory bank 0x%x info present %d return code %d\n",
memoryBank, memoryInfo != nullptr, ret);
}
return ret;
}
PhysicalDevicePciSpeedInfo Drm::getPciSpeedInfo() const {
PhysicalDevicePciSpeedInfo pciSpeedInfo = {};
std::string pathPrefix{};
bool isIntegratedDevice = rootDeviceEnvironment.getHardwareInfo()->capabilityTable.isIntegratedDevice;
// If integrated device, read properties from the specific device path.
// If discrete device, read properties from the root path of the pci device.
if (isIntegratedDevice) {
auto devicePath = NEO::getPciLinkPath(getFileDescriptor());
if (!devicePath.has_value()) {
return pciSpeedInfo;
}
pathPrefix = "/sys/class/drm/" + devicePath.value() + "/device/";
} else {
auto rootPath = NEO::getPciRootPath(getFileDescriptor());
if (!rootPath.has_value()) {
return pciSpeedInfo;
}
pathPrefix += "/sys/devices" + rootPath.value();
}
std::array<char, 32> readString = {'\0'};
errno = 0;
auto readFile = [](const std::string fileName, const std::string_view pathPrefix, std::array<char, 32> &readString) {
std::ostringstream linkWidthStream{};
linkWidthStream << pathPrefix << fileName;
int fd = NEO::SysCalls::open(linkWidthStream.str().c_str(), O_RDONLY);
if (fd < 0) {
return false;
}
ssize_t bytesRead = NEO::SysCalls::pread(fd, readString.data(), readString.size() - 1, 0);
NEO::SysCalls::close(fd);
if (bytesRead <= 0) {
return false;
}
std::replace(readString.begin(), readString.end(), '\n', '\0');
return true;
};
// read max link width
if (readFile("/max_link_width", pathPrefix, readString) != true) {
return pciSpeedInfo;
}
char *endPtr = nullptr;
uint32_t linkWidth = static_cast<uint32_t>(std::strtoul(readString.data(), &endPtr, 10));
if ((endPtr == readString.data()) || (errno != 0)) {
return pciSpeedInfo;
}
pciSpeedInfo.width = linkWidth;
// read max link speed
if (readFile("/max_link_speed", pathPrefix, readString) != true) {
return pciSpeedInfo;
}
endPtr = nullptr;
const auto maxSpeed = strtod(readString.data(), &endPtr);
if ((endPtr == readString.data()) || (errno != 0)) {
return pciSpeedInfo;
}
double gen3EncodingLossFactor = 128.0 / 130.0;
std::map<double, std::pair<int32_t, double>> maxSpeedToGenAndEncodingLossMapping{
//{max link speed, {pci generation, encoding loss factor}}
{2.5, {1, 0.2}},
{5.0, {2, 0.2}},
{8.0, {3, gen3EncodingLossFactor}},
{16.0, {4, gen3EncodingLossFactor}},
{32.0, {5, gen3EncodingLossFactor}}};
if (maxSpeedToGenAndEncodingLossMapping.find(maxSpeed) == maxSpeedToGenAndEncodingLossMapping.end()) {
return pciSpeedInfo;
}
pciSpeedInfo.genVersion = maxSpeedToGenAndEncodingLossMapping[maxSpeed].first;
constexpr double gigaBitsPerSecondToBytesPerSecondMultiplier = 125000000;
const auto maxSpeedWithEncodingLoss = maxSpeed * gigaBitsPerSecondToBytesPerSecondMultiplier * maxSpeedToGenAndEncodingLossMapping[maxSpeed].second;
pciSpeedInfo.maxBandwidth = static_cast<int64_t>(maxSpeedWithEncodingLoss * pciSpeedInfo.width);
return pciSpeedInfo;
}
int Drm::waitOnUserFences(OsContextLinux &osContext, uint64_t address, uint64_t value, uint32_t numActiveTiles, int64_t timeout, uint32_t postSyncOffset, bool userInterrupt,
uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) {
int ret = waitOnUserFencesImpl(static_cast<const OsContextLinux &>(osContext), address, value, numActiveTiles,
timeout, postSyncOffset, userInterrupt, externalInterruptId, allocForInterruptWait);
if (ret != 0 && getErrno() == EIO && checkGpuPageFaultRequired()) {
checkResetStatus(osContext);
}
return ret;
}
int Drm::waitOnUserFencesImpl(const OsContextLinux &osContext, uint64_t address, uint64_t value, uint32_t numActiveTiles, int64_t timeout, uint32_t postSyncOffset, bool userInterrupt,
uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) {
auto &drmContextIds = osContext.getDrmContextIds();
UNRECOVERABLE_IF(numActiveTiles > drmContextIds.size());
auto completionFenceCpuAddress = address;
const auto selectedTimeout = osContext.isHangDetected() ? 1 : timeout;
for (auto drmIterator = 0u; drmIterator < numActiveTiles; drmIterator++) {
if (*reinterpret_cast<uint32_t *>(completionFenceCpuAddress) < value) {
static constexpr uint16_t flags = 0;
int retVal = waitUserFence(drmContextIds[drmIterator], completionFenceCpuAddress, value, Drm::ValueWidth::u64, selectedTimeout, flags, userInterrupt, externalInterruptId, allocForInterruptWait);
if (debugManager.flags.PrintCompletionFenceUsage.get()) {
std::cout << "Completion fence waited."
<< " Status: " << retVal
<< ", CPU address: " << std::hex << completionFenceCpuAddress << std::dec
<< ", current value: " << *reinterpret_cast<uint32_t *>(completionFenceCpuAddress)
<< ", wait value: " << value << std::endl;
}
if (retVal != 0) {
return retVal;
}
} else if (debugManager.flags.PrintCompletionFenceUsage.get()) {
std::cout << "Completion fence already completed."
<< " CPU address: " << std::hex << completionFenceCpuAddress << std::dec
<< ", current value: " << *reinterpret_cast<uint32_t *>(completionFenceCpuAddress)
<< ", wait value: " << value << std::endl;
}
if (externalInterruptId != NEO::InterruptId::notUsed) {
break;
}
completionFenceCpuAddress = ptrOffset(completionFenceCpuAddress, postSyncOffset);
}
return 0;
}
const HardwareInfo *Drm::getHardwareInfo() const { return rootDeviceEnvironment.getHardwareInfo(); }
uint64_t Drm::alignUpGttSize(uint64_t inputGttSize) {
constexpr uint64_t gttSize47bit = (1ull << 47);
constexpr uint64_t gttSize48bit = (1ull << 48);
if (inputGttSize > gttSize47bit && inputGttSize < gttSize48bit) {
return gttSize48bit;
}
return inputGttSize;
}
bool Drm::isDrmSupported(int fileDescriptor) {
auto drmVersion = Drm::getDrmVersion(fileDescriptor);
return "i915" == drmVersion || "xe" == drmVersion;
}
bool Drm::queryDeviceIdAndRevision() {
auto drmVersion = Drm::getDrmVersion(getFileDescriptor());
if ("xe" == drmVersion) {
this->setPerContextVMRequired(false);
return IoctlHelperXe::queryDeviceIdAndRevision(*this);
}
return IoctlHelperI915::queryDeviceIdAndRevision(*this);
}
uint32_t Drm::getAggregatedProcessCount() const {
return ioctlHelper->getNumProcesses();
}
template std::vector<uint16_t> Drm::query<uint16_t>(uint32_t queryId, uint32_t queryItemFlags);
template std::vector<uint32_t> Drm::query<uint32_t>(uint32_t queryId, uint32_t queryItemFlags);
template std::vector<uint64_t> Drm::query<uint64_t>(uint32_t queryId, uint32_t queryItemFlags);
} // namespace NEO
Reference in New Issue View Git Blame Copy Permalink