intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-21 09:14:47 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
48d6788b3a0db85264045c53695b31966283c55a
compute-runtime/shared/source/os_interface/linux/drm_neo.cpp
Mateusz Jablonski 48d6788b3a fix: add fallback for missing eu count in topology 
for xe kmd there will be new query for EU per DSS for PVC, LNL, BMG platforms
when new query is available, previous one (currently used in NEO) will be empty
To avoid integration issues this commit adds fallback to setup that value based
on max eu per dss that we get from GuC in device blob

Support for new query is in PR https://github.com/intel/compute-runtime/pull/745

Related-To: NEO-12012
Signed-off-by: Mateusz Jablonski <mateusz.jablonski@intel.com>
2024-07-17 14:31:29 +02:00

1671 lines
64 KiB
C++
Raw Blame History

/*
* Copyright (C) 2018-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/os_interface/linux/drm_neo.h"
#include "shared/source/command_stream/submission_status.h"
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/execution_environment/execution_environment.h"
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/gmm_helper/cache_settings_helper.h"
#include "shared/source/gmm_helper/client_context/gmm_client_context.h"
#include "shared/source/gmm_helper/gmm.h"
#include "shared/source/gmm_helper/resource_info.h"
#include "shared/source/helpers/aligned_memory.h"
#include "shared/source/helpers/basic_math.h"
#include "shared/source/helpers/constants.h"
#include "shared/source/helpers/debug_helpers.h"
#include "shared/source/helpers/gfx_core_helper.h"
#include "shared/source/helpers/gpu_page_fault_helper.h"
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/os_interface/driver_info.h"
#include "shared/source/os_interface/linux/cache_info.h"
#include "shared/source/os_interface/linux/drm_buffer_object.h"
#include "shared/source/os_interface/linux/drm_engine_mapper.h"
#include "shared/source/os_interface/linux/drm_gem_close_worker.h"
#include "shared/source/os_interface/linux/drm_memory_manager.h"
#include "shared/source/os_interface/linux/drm_memory_operations_handler_bind.h"
#include "shared/source/os_interface/linux/drm_wrappers.h"
#include "shared/source/os_interface/linux/engine_info.h"
#include "shared/source/os_interface/linux/hw_device_id.h"
#include "shared/source/os_interface/linux/ioctl_helper.h"
#include "shared/source/os_interface/linux/memory_info.h"
#include "shared/source/os_interface/linux/os_context_linux.h"
#include "shared/source/os_interface/linux/os_inc.h"
#include "shared/source/os_interface/linux/pci_path.h"
#include "shared/source/os_interface/linux/sys_calls.h"
#include "shared/source/os_interface/linux/system_info.h"
#include "shared/source/os_interface/os_environment.h"
#include "shared/source/os_interface/os_interface.h"
#include "shared/source/os_interface/product_helper.h"
#include "shared/source/release_helper/release_helper.h"
#include "shared/source/utilities/api_intercept.h"
#include "shared/source/utilities/directory.h"
#include "shared/source/utilities/io_functions.h"
#include <cstdio>
#include <cstring>
#include <fcntl.h>
#include <fstream>
#include <map>
#include <sstream>
namespace NEO {
Drm::Drm(std::unique_ptr<HwDeviceIdDrm> &&hwDeviceIdIn, RootDeviceEnvironment &rootDeviceEnvironment)
: DriverModel(DriverModelType::drm),
hwDeviceId(std::move(hwDeviceIdIn)), rootDeviceEnvironment(rootDeviceEnvironment) {
pagingFence.fill(0u);
fenceVal.fill(0u);
}
SubmissionStatus Drm::getSubmissionStatusFromReturnCode(int32_t retCode) {
switch (retCode) {
case 0:
return SubmissionStatus::success;
case EWOULDBLOCK:
case ENOMEM:
case ENOSPC:
return SubmissionStatus::outOfHostMemory;
case ENXIO:
return SubmissionStatus::outOfMemory;
default:
return SubmissionStatus::failed;
}
}
void Drm::queryAndSetVmBindPatIndexProgrammingSupport() {
auto &productHelper = rootDeviceEnvironment.getHelper<ProductHelper>();
this->vmBindPatIndexProgrammingSupported = productHelper.isVmBindPatIndexProgrammingSupported();
}
int Drm::ioctl(DrmIoctl request, void *arg) {
auto requestValue = getIoctlRequestValue(request, ioctlHelper.get());
int ret;
int returnedErrno = 0;
SYSTEM_ENTER();
do {
auto measureTime = debugManager.flags.PrintKmdTimes.get();
std::chrono::steady_clock::time_point start;
std::chrono::steady_clock::time_point end;
auto printIoctl = debugManager.flags.PrintIoctlEntries.get();
if (printIoctl) {
printf("IOCTL %s called\n", getIoctlString(request, ioctlHelper.get()).c_str());
}
if (measureTime) {
start = std::chrono::steady_clock::now();
}
ret = SysCalls::ioctl(getFileDescriptor(), requestValue, arg);
if (ret != 0) {
returnedErrno = getErrno();
}
if (measureTime) {
end = std::chrono::steady_clock::now();
long long elapsedTime = std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count();
IoctlStatisticsEntry ioctlData{};
auto ioctlDataIt = this->ioctlStatistics.find(request);
if (ioctlDataIt != this->ioctlStatistics.end()) {
ioctlData = ioctlDataIt->second;
}
ioctlData.totalTime += elapsedTime;
ioctlData.count++;
ioctlData.minTime = std::min(ioctlData.minTime, elapsedTime);
ioctlData.maxTime = std::max(ioctlData.maxTime, elapsedTime);
this->ioctlStatistics[request] = ioctlData;
}
if (printIoctl) {
if (ret == 0) {
printf("IOCTL %s returns %d\n",
getIoctlString(request, ioctlHelper.get()).c_str(), ret);
} else {
printf("IOCTL %s returns %d, errno %d(%s)\n",
getIoctlString(request, ioctlHelper.get()).c_str(), ret, returnedErrno, strerror(returnedErrno));
}
}
} while (ret == -1 && checkIfIoctlReinvokeRequired(returnedErrno, request, ioctlHelper.get()));
SYSTEM_LEAVE(request);
return ret;
}
int Drm::getParamIoctl(DrmParam param, int *dstValue) {
GetParam getParam{};
getParam.param = getDrmParamValue(param, ioctlHelper.get());
getParam.value = dstValue;
int retVal = ioctlHelper ? ioctlHelper->ioctl(DrmIoctl::getparam, &getParam) : ioctl(DrmIoctl::getparam, &getParam);
if (debugManager.flags.PrintIoctlEntries.get()) {
printf("DRM_IOCTL_I915_GETPARAM: param: %s, output value: %d, retCode:% d\n",
getDrmParamString(param, ioctlHelper.get()).c_str(),
*getParam.value,
retVal);
}
return retVal;
}
bool Drm::queryI915DeviceIdAndRevision() {
HardwareInfo *hwInfo = rootDeviceEnvironment.getMutableHardwareInfo();
int deviceId = hwInfo->platform.usDeviceID;
int revisionId = hwInfo->platform.usRevId;
auto ret = getParamIoctl(DrmParam::paramChipsetId, &deviceId);
if (ret != 0) {
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Cannot query device ID parameter!\n");
return false;
}
ret = getParamIoctl(DrmParam::paramRevision, &revisionId);
if (ret != 0) {
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Cannot query device Rev ID parameter!\n");
return false;
}
hwInfo->platform.usDeviceID = deviceId;
hwInfo->platform.usRevId = revisionId;
return true;
}
int Drm::enableTurboBoost() {
GemContextParam contextParam = {};
contextParam.param = contextPrivateParamBoost;
contextParam.value = 1;
return ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
}
int Drm::getEnabledPooledEu(int &enabled) {
return getParamIoctl(DrmParam::paramHasPooledEu, &enabled);
}
std::string Drm::getSysFsPciPath() {
std::string path = std::string(Os::sysFsPciPathPrefix) + hwDeviceId->getPciPath() + "/drm";
std::string expectedFilePrefix = path + "/card";
auto files = Directory::getFiles(path.c_str());
for (auto &file : files) {
if (file.find(expectedFilePrefix.c_str()) != std::string::npos) {
return file;
}
}
return {};
}
bool Drm::readSysFsAsString(const std::string &relativeFilePath, std::string &readString) {
auto devicePath = getSysFsPciPath();
if (devicePath.empty()) {
return false;
}
const std::string fileName = devicePath + relativeFilePath;
int fd = SysCalls::open(fileName.c_str(), O_RDONLY);
if (fd < 0) {
return false;
}
ssize_t bytesRead = SysCalls::pread(fd, readString.data(), readString.size() - 1, 0);
NEO::SysCalls::close(fd);
if (bytesRead <= 0) {
return false;
}
std::replace(readString.begin(), readString.end(), '\n', '\0');
return true;
}
int Drm::queryGttSize(uint64_t &gttSizeOutput, bool alignUpToFullRange) {
GemContextParam contextParam = {0};
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamGttSize);
int ret = ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &contextParam);
if (ret == 0) {
if (alignUpToFullRange) {
gttSizeOutput = Drm::alignUpGttSize(contextParam.value);
} else {
gttSizeOutput = contextParam.value;
}
}
return ret;
}
bool Drm::isGpuHangDetected(OsContext &osContext) {
bool ret = checkResetStatus(osContext);
auto threshold = getGpuFaultCheckThreshold();
if (checkGpuPageFaultRequired()) {
if (gpuFaultCheckCounter >= threshold) {
auto memoryManager = static_cast<DrmMemoryManager *>(this->rootDeviceEnvironment.executionEnvironment.memoryManager.get());
memoryManager->checkUnexpectedGpuPageFault();
gpuFaultCheckCounter = 0;
return false;
}
gpuFaultCheckCounter++;
}
return ret;
}
bool Drm::checkResetStatus(OsContext &osContext) {
const auto osContextLinux = static_cast<OsContextLinux *>(&osContext);
const auto &drmContextIds = osContextLinux->getDrmContextIds();
for (const auto drmContextId : drmContextIds) {
ResetStats resetStats{};
resetStats.contextId = drmContextId;
ResetStatsFault fault{};
uint32_t status = 0;
const auto retVal{ioctlHelper->getResetStats(resetStats, &status, &fault)};
UNRECOVERABLE_IF(retVal != 0);
if (checkToDisableScratchPage() && ioctlHelper->validPageFault(fault.flags)) {
bool banned = ((status & ioctlHelper->getStatusForResetStats(true)) != 0);
IoFunctions::fprintf(stderr, "FATAL: Unexpected page fault from GPU at 0x%llx, ctx_id: %u (%s) type: %d (%s), level: %d (%s), access: %d (%s), banned: %d, aborting.\n",
fault.addr,
resetStats.contextId,
EngineHelpers::engineTypeToString(osContext.getEngineType()).c_str(),
fault.type, GpuPageFaultHelpers::faultTypeToString(static_cast<FaultType>(fault.type)).c_str(),
fault.level, GpuPageFaultHelpers::faultLevelToString(static_cast<FaultLevel>(fault.level)).c_str(),
fault.access, GpuPageFaultHelpers::faultAccessToString(static_cast<FaultAccess>(fault.access)).c_str(),
banned);
IoFunctions::fprintf(stdout, "FATAL: Unexpected page fault from GPU at 0x%llx, ctx_id: %u (%s) type: %d (%s), level: %d (%s), access: %d (%s), banned: %d, aborting.\n",
fault.addr,
resetStats.contextId,
EngineHelpers::engineTypeToString(osContext.getEngineType()).c_str(),
fault.type, GpuPageFaultHelpers::faultTypeToString(static_cast<FaultType>(fault.type)).c_str(),
fault.level, GpuPageFaultHelpers::faultLevelToString(static_cast<FaultLevel>(fault.level)).c_str(),
fault.access, GpuPageFaultHelpers::faultAccessToString(static_cast<FaultAccess>(fault.access)).c_str(),
banned);
UNRECOVERABLE_IF(true);
}
if (resetStats.batchActive > 0 || resetStats.batchPending > 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "ERROR: GPU HANG detected!\n");
osContextLinux->setHangDetected();
return true;
}
}
return false;
}
void Drm::checkPreemptionSupport() {
preemptionSupported = ioctlHelper->isPreemptionSupported();
}
void Drm::checkQueueSliceSupport() {
sliceCountChangeSupported = getQueueSliceCount(&sseu) == 0 ? true : false;
}
void Drm::setLowPriorityContextParam(uint32_t drmContextId) {
GemContextParam gcp = {};
gcp.contextId = drmContextId;
gcp.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamPriority);
gcp.value = -1023;
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &gcp);
UNRECOVERABLE_IF(retVal != 0);
}
int Drm::getQueueSliceCount(GemContextParamSseu *sseu) {
GemContextParam contextParam = {};
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamSseu);
sseu->engine.engineClass = ioctlHelper->getDrmParamValue(DrmParam::engineClassRender);
sseu->engine.engineInstance = ioctlHelper->getDrmParamValue(DrmParam::execDefault);
contextParam.value = reinterpret_cast<uint64_t>(sseu);
contextParam.size = sizeof(struct GemContextParamSseu);
return ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &contextParam);
}
uint64_t Drm::getSliceMask(uint64_t sliceCount) {
return maxNBitValue(sliceCount);
}
bool Drm::setQueueSliceCount(uint64_t sliceCount) {
if (sliceCountChangeSupported) {
GemContextParam contextParam = {};
sseu.sliceMask = getSliceMask(sliceCount);
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamSseu);
contextParam.contextId = 0;
contextParam.value = reinterpret_cast<uint64_t>(&sseu);
contextParam.size = sizeof(struct GemContextParamSseu);
int retVal = ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
if (retVal == 0) {
return true;
}
}
return false;
}
void Drm::checkNonPersistentContextsSupport() {
GemContextParam contextParam = {};
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamPersistence);
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &contextParam);
if (retVal == 0 && contextParam.value == 1) {
nonPersistentContextsSupported = true;
} else {
nonPersistentContextsSupported = false;
}
}
void Drm::setNonPersistentContext(uint32_t drmContextId) {
GemContextParam contextParam = {};
contextParam.contextId = drmContextId;
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamPersistence);
ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
}
void Drm::setUnrecoverableContext(uint32_t drmContextId) {
GemContextParam contextParam = {};
contextParam.contextId = drmContextId;
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamRecoverable);
contextParam.value = 0;
contextParam.size = 0;
ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
}
int Drm::createDrmContext(uint32_t drmVmId, bool isDirectSubmissionRequested, bool isCooperativeContextRequested) {
GemContextCreateExt gcc{};
if (debugManager.flags.DirectSubmissionDrmContext.get() != -1) {
isDirectSubmissionRequested = debugManager.flags.DirectSubmissionDrmContext.get();
}
if (isDirectSubmissionRequested) {
gcc.flags |= ioctlHelper->getDirectSubmissionFlag();
}
GemContextCreateExtSetParam extSetparam = {};
if (drmVmId > 0) {
extSetparam.base.name = ioctlHelper->getDrmParamValue(DrmParam::contextCreateExtSetparam);
extSetparam.param.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamVm);
extSetparam.param.value = drmVmId;
gcc.extensions = reinterpret_cast<uint64_t>(&extSetparam);
gcc.flags |= ioctlHelper->getDrmParamValue(DrmParam::contextCreateFlagsUseExtensions);
}
if (debugManager.flags.CreateContextWithAccessCounters.get() > 0) {
return ioctlHelper->createContextWithAccessCounters(gcc);
}
if (debugManager.flags.ForceRunAloneContext.get() != -1) {
isCooperativeContextRequested = debugManager.flags.ForceRunAloneContext.get();
}
if (isCooperativeContextRequested) {
return ioctlHelper->createCooperativeContext(gcc);
}
auto ioctlResult = ioctlHelper->ioctl(DrmIoctl::gemContextCreateExt, &gcc);
if (ioctlResult < 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "WARNING: GemContextCreateExt ioctl failed. Not exposing this root device\n");
return ioctlResult;
}
return gcc.contextId;
}
void Drm::destroyDrmContext(uint32_t drmContextId) {
GemContextDestroy destroy{};
destroy.contextId = drmContextId;
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextDestroy, &destroy);
UNRECOVERABLE_IF((retVal != 0) && (errno != ENODEV));
}
void Drm::destroyDrmVirtualMemory(uint32_t drmVmId) {
GemVmControl ctl = {};
ctl.vmId = drmVmId;
auto ret = ioctlHelper->ioctl(DrmIoctl::gemVmDestroy, &ctl);
UNRECOVERABLE_IF((ret != 0) && (errno != ENODEV));
}
int Drm::queryVmId(uint32_t drmContextId, uint32_t &vmId) {
GemContextParam param{};
param.contextId = drmContextId;
param.value = 0;
param.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamVm);
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &param);
vmId = static_cast<uint32_t>(param.value);
return retVal;
}
std::unique_lock<std::mutex> Drm::lockBindFenceMutex() {
return std::unique_lock<std::mutex>(this->bindFenceMutex);
}
int Drm::getEuTotal(int &euTotal) {
return getParamIoctl(DrmParam::paramEuTotal, &euTotal);
}
int Drm::getSubsliceTotal(int &subsliceTotal) {
return getParamIoctl(DrmParam::paramSubsliceTotal, &subsliceTotal);
}
int Drm::getMinEuInPool(int &minEUinPool) {
return getParamIoctl(DrmParam::paramMinEuInPool, &minEUinPool);
}
int Drm::getErrno() {
return errno;
}
int Drm::setupHardwareInfo(const DeviceDescriptor *device, bool setupFeatureTableAndWorkaroundTable) {
const auto usDeviceIdOverride = rootDeviceEnvironment.getHardwareInfo()->platform.usDeviceID;
const auto usRevIdOverride = rootDeviceEnvironment.getHardwareInfo()->platform.usRevId;
// reset hwInfo and apply overrides
rootDeviceEnvironment.setHwInfo(device->pHwInfo);
HardwareInfo *hwInfo = rootDeviceEnvironment.getMutableHardwareInfo();
hwInfo->platform.usDeviceID = usDeviceIdOverride;
hwInfo->platform.usRevId = usRevIdOverride;
rootDeviceEnvironment.initProductHelper();
rootDeviceEnvironment.initGfxCoreHelper();
rootDeviceEnvironment.initApiGfxCoreHelper();
rootDeviceEnvironment.initCompilerProductHelper();
rootDeviceEnvironment.initAilConfigurationHelper();
auto result = rootDeviceEnvironment.initAilConfiguration();
if (false == result) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: AIL creation failed!\n");
return -1;
}
const auto productFamily = hwInfo->platform.eProductFamily;
setupIoctlHelper(productFamily);
ioctlHelper->setupIpVersion();
rootDeviceEnvironment.initReleaseHelper();
DrmQueryTopologyData topologyData = {};
bool status = queryTopology(*hwInfo, topologyData);
if (!status) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "WARNING: Topology query failed!\n");
auto ret = getEuTotal(topologyData.euCount);
if (ret != 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Cannot query EU total parameter!\n");
return ret;
}
ret = getSubsliceTotal(topologyData.subSliceCount);
if (ret != 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Cannot query subslice total parameter!\n");
return ret;
}
}
hwInfo->gtSystemInfo.SliceCount = static_cast<uint32_t>(topologyData.sliceCount);
hwInfo->gtSystemInfo.SubSliceCount = static_cast<uint32_t>(topologyData.subSliceCount);
hwInfo->gtSystemInfo.DualSubSliceCount = static_cast<uint32_t>(topologyData.subSliceCount);
hwInfo->gtSystemInfo.EUCount = static_cast<uint32_t>(topologyData.euCount);
auto releaseHelper = rootDeviceEnvironment.getReleaseHelper();
auto numThreadsPerEu = releaseHelper ? releaseHelper->getNumThreadsPerEu() : 7u;
hwInfo->gtSystemInfo.ThreadCount = numThreadsPerEu * hwInfo->gtSystemInfo.EUCount;
if (topologyData.maxSubSliceCount > 0) {
hwInfo->gtSystemInfo.MaxSubSlicesSupported = static_cast<uint32_t>(topologyData.maxSubSliceCount);
hwInfo->gtSystemInfo.MaxDualSubSlicesSupported = static_cast<uint32_t>(topologyData.maxSubSliceCount);
}
if (topologyData.numL3Banks > 0) {
hwInfo->gtSystemInfo.L3BankCount = topologyData.numL3Banks;
}
if (hwInfo->gtSystemInfo.MaxSlicesSupported == 0) {
hwInfo->gtSystemInfo.MaxSlicesSupported = hwInfo->gtSystemInfo.SliceCount;
}
if (hwInfo->gtSystemInfo.MaxEuPerSubSlice == 0 && hwInfo->gtSystemInfo.SubSliceCount != 0) {
hwInfo->gtSystemInfo.MaxEuPerSubSlice = hwInfo->gtSystemInfo.EUCount / hwInfo->gtSystemInfo.SubSliceCount;
}
status = querySystemInfo();
device->setupHardwareInfo(hwInfo, setupFeatureTableAndWorkaroundTable, releaseHelper);
rootDeviceEnvironment.setRcsExposure();
if (status) {
systemInfo->checkSysInfoMismatch(hwInfo);
setupSystemInfo(hwInfo, systemInfo.get());
uint32_t bankCount = (hwInfo->gtSystemInfo.L3BankCount > 0) ? hwInfo->gtSystemInfo.L3BankCount : hwInfo->gtSystemInfo.MaxDualSubSlicesSupported;
hwInfo->gtSystemInfo.L3CacheSizeInKb = systemInfo->getL3BankSizeInKb() * bankCount;
}
if (!hwInfo->gtSystemInfo.EUCount) {
return -1;
}
setupCacheInfo(*hwInfo);
hwInfo->capabilityTable.deviceName = device->devName;
return 0;
}
void appendHwDeviceId(std::vector<std::unique_ptr<HwDeviceId>> &hwDeviceIds, int fileDescriptor, const char *pciPath, const char *devNodePath) {
if (fileDescriptor >= 0) {
if (Drm::isDrmSupported(fileDescriptor)) {
hwDeviceIds.push_back(std::make_unique<HwDeviceIdDrm>(fileDescriptor, pciPath, devNodePath));
} else {
SysCalls::close(fileDescriptor);
}
}
}
std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevices(ExecutionEnvironment &executionEnvironment) {
std::string str = "";
return Drm::discoverDevices(executionEnvironment, str);
}
std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevice(ExecutionEnvironment &executionEnvironment, std::string &osPciPath) {
return Drm::discoverDevices(executionEnvironment, osPciPath);
}
std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevices(ExecutionEnvironment &executionEnvironment, std::string &osPciPath) {
std::vector<std::unique_ptr<HwDeviceId>> hwDeviceIds;
executionEnvironment.osEnvironment = std::make_unique<OsEnvironment>();
size_t numRootDevices = 0u;
if (debugManager.flags.CreateMultipleRootDevices.get()) {
numRootDevices = debugManager.flags.CreateMultipleRootDevices.get();
}
std::vector<std::string> files = Directory::getFiles(Os::pciDevicesDirectory);
if (files.size() == 0) {
const char *pathPrefix = "/dev/dri/renderD";
const unsigned int maxDrmDevices = 64;
unsigned int startNum = 128;
for (unsigned int i = 0; i < maxDrmDevices; i++) {
std::string path = std::string(pathPrefix) + std::to_string(i + startNum);
int fileDescriptor = SysCalls::open(path.c_str(), O_RDWR | O_CLOEXEC);
if (fileDescriptor < 0) {
continue;
}
auto pciPath = NEO::getPciPath(fileDescriptor);
appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.value_or("0000:00:02.0").c_str(), path.c_str());
if (!hwDeviceIds.empty() && hwDeviceIds.size() == numRootDevices) {
break;
}
}
return hwDeviceIds;
}
do {
const char *renderDeviceSuffix = "-render";
for (std::vector<std::string>::iterator file = files.begin(); file != files.end(); ++file) {
std::string_view devicePathView(file->c_str(), file->size());
devicePathView = devicePathView.substr(strlen(Os::pciDevicesDirectory));
auto rdsPos = devicePathView.rfind(renderDeviceSuffix);
if (rdsPos == std::string::npos) {
continue;
}
if (rdsPos < devicePathView.size() - strlen(renderDeviceSuffix)) {
continue;
}
// at least 'pci-0000:00:00.0' -> 16
if (rdsPos < 16 || devicePathView[rdsPos - 13] != '-') {
continue;
}
std::string pciPath(devicePathView.substr(rdsPos - 12, 12));
if (!osPciPath.empty()) {
if (osPciPath.compare(pciPath) != 0) {
// if osPciPath is non-empty, then interest is only in discovering device having same bdf as ocPciPath. Skip all other devices.
continue;
}
}
if (debugManager.flags.FilterBdfPath.get() != "unk") {
if (devicePathView.find(debugManager.flags.FilterBdfPath.get().c_str()) == std::string::npos) {
continue;
}
}
int fileDescriptor = SysCalls::open(file->c_str(), O_RDWR | O_CLOEXEC);
appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.c_str(), file->c_str());
if (!hwDeviceIds.empty() && hwDeviceIds.size() == numRootDevices) {
break;
}
}
if (hwDeviceIds.empty()) {
return hwDeviceIds;
}
} while (hwDeviceIds.size() < numRootDevices);
return hwDeviceIds;
}
std::string Drm::getDrmVersion(int fileDescriptor) {
DrmVersion version = {};
char name[5] = {};
version.name = name;
version.nameLen = 5;
auto requestValue = getIoctlRequestValue(DrmIoctl::version, nullptr);
int ret = SysCalls::ioctl(fileDescriptor, requestValue, &version);
if (ret) {
return {};
}
name[4] = '\0';
return std::string(name);
}
template <typename DataType>
std::vector<DataType> Drm::query(uint32_t queryId, uint32_t queryItemFlags) {
Query query{};
QueryItem queryItem{};
queryItem.queryId = queryId;
queryItem.length = 0; // query length first
queryItem.flags = queryItemFlags;
query.itemsPtr = reinterpret_cast<uint64_t>(&queryItem);
query.numItems = 1;
auto ret = ioctlHelper->ioctl(DrmIoctl::query, &query);
if (ret != 0 || queryItem.length <= 0) {
return {};
}
auto data = std::vector<DataType>(Math::divideAndRoundUp(queryItem.length, sizeof(DataType)), 0);
queryItem.dataPtr = castToUint64(data.data());
ret = ioctlHelper->ioctl(DrmIoctl::query, &query);
if (ret != 0 || queryItem.length <= 0) {
return {};
}
return data;
}
void Drm::printIoctlStatistics() {
if (!debugManager.flags.PrintKmdTimes.get()) {
return;
}
printf("\n--- Ioctls statistics ---\n");
printf("%41s %15s %10s %20s %20s %20s", "Request", "Total time(ns)", "Count", "Avg time per ioctl", "Min", "Max\n");
for (const auto &ioctlData : this->ioctlStatistics) {
printf("%41s %15llu %10lu %20f %20lld %20lld\n",
getIoctlString(ioctlData.first, ioctlHelper.get()).c_str(),
ioctlData.second.totalTime,
static_cast<unsigned long>(ioctlData.second.count),
ioctlData.second.totalTime / static_cast<double>(ioctlData.second.count),
ioctlData.second.minTime,
ioctlData.second.maxTime);
}
printf("\n");
}
bool Drm::createVirtualMemoryAddressSpace(uint32_t vmCount) {
for (auto i = 0u; i < vmCount; i++) {
uint32_t id = i;
if (0 != createDrmVirtualMemory(id)) {
return false;
}
virtualMemoryIds.push_back(id);
}
return true;
}
void Drm::destroyVirtualMemoryAddressSpace() {
for (auto id : virtualMemoryIds) {
destroyDrmVirtualMemory(id);
}
virtualMemoryIds.clear();
}
uint32_t Drm::getVirtualMemoryAddressSpace(uint32_t vmId) const {
if (vmId < virtualMemoryIds.size()) {
return virtualMemoryIds[vmId];
}
return 0;
}
void Drm::setNewResourceBoundToVM(BufferObject *bo, uint32_t vmHandleId) {
if (!this->rootDeviceEnvironment.getProductHelper().isTlbFlushRequired()) {
return;
}
const auto &engines = this->rootDeviceEnvironment.executionEnvironment.memoryManager->getRegisteredEngines(bo->getRootDeviceIndex());
for (const auto &engine : engines) {
if (engine.osContext->getDeviceBitfield().test(vmHandleId)) {
auto osContextLinux = static_cast<OsContextLinux *>(engine.osContext);
osContextLinux->setNewResourceBound();
}
}
}
PhysicalDevicePciBusInfo Drm::getPciBusInfo() const {
PhysicalDevicePciBusInfo pciBusInfo(PhysicalDevicePciBusInfo::invalidValue, PhysicalDevicePciBusInfo::invalidValue, PhysicalDevicePciBusInfo::invalidValue, PhysicalDevicePciBusInfo::invalidValue);
if (adapterBDF.Data != std::numeric_limits<uint32_t>::max()) {
pciBusInfo.pciDomain = this->pciDomain;
pciBusInfo.pciBus = adapterBDF.Bus;
pciBusInfo.pciDevice = adapterBDF.Device;
pciBusInfo.pciFunction = adapterBDF.Function;
}
return pciBusInfo;
}
void Drm::cleanup() {
destroyVirtualMemoryAddressSpace();
}
Drm::~Drm() {
this->printIoctlStatistics();
}
int Drm::queryAdapterBDF() {
constexpr int pciBusInfoTokensNum = 4;
uint16_t domain = -1;
uint8_t bus = -1, device = -1, function = -1;
if (NEO::parseBdfString(hwDeviceId->getPciPath(), domain, bus, device, function) != pciBusInfoTokensNum) {
adapterBDF.Data = std::numeric_limits<uint32_t>::max();
return 1;
}
setPciDomain(domain);
adapterBDF.Bus = bus;
adapterBDF.Function = function;
adapterBDF.Device = device;
return 0;
}
void Drm::setGmmInputArgs(void *args) {
auto gmmInArgs = reinterpret_cast<GMM_INIT_IN_ARGS *>(args);
#if defined(__linux__)
gmmInArgs->FileDescriptor = adapterBDF.Data;
#endif
gmmInArgs->ClientType = GMM_CLIENT::GMM_OCL_VISTA;
}
const std::vector<int> &Drm::getSliceMappings(uint32_t deviceIndex) {
return topologyMap[deviceIndex].sliceIndices;
}
int Drm::waitHandle(uint32_t waitHandle, int64_t timeout) {
UNRECOVERABLE_IF(isVmBindAvailable());
GemWait wait{};
wait.boHandle = waitHandle;
wait.timeoutNs = timeout;
int ret = ioctlHelper->ioctl(DrmIoctl::gemWait, &wait);
if (ret != 0) {
int err = errno;
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(I915_GEM_WAIT) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
}
return ret;
}
int Drm::getTimestampFrequency(int &frequency) {
frequency = 0;
return getParamIoctl(DrmParam::paramCsTimestampFrequency, &frequency);
}
int Drm::getOaTimestampFrequency(int &frequency) {
frequency = 0;
return getParamIoctl(DrmParam::paramOATimestampFrequency, &frequency);
}
bool Drm::queryEngineInfo() {
return Drm::queryEngineInfo(false);
}
bool Drm::sysmanQueryEngineInfo() {
return Drm::queryEngineInfo(true);
}
bool Drm::isDebugAttachAvailable() {
int enableEuDebug = getEuDebugSysFsEnable();
return (enableEuDebug == 1) && ioctlHelper->isDebugAttachAvailable();
}
int Drm::getEuDebugSysFsEnable() {
return ioctlHelper->getEuDebugSysFsEnable();
}
int getMaxGpuFrequencyOfDevice(Drm &drm, std::string &sysFsPciPath, int &maxGpuFrequency) {
maxGpuFrequency = 0;
std::string clockSysFsPath = sysFsPciPath + drm.getIoctlHelper()->getFileForMaxGpuFrequency();
std::ifstream ifs(clockSysFsPath.c_str(), std::ifstream::in);
if (ifs.fail()) {
return -1;
}
ifs >> maxGpuFrequency;
ifs.close();
return 0;
}
int getMaxGpuFrequencyOfSubDevice(Drm &drm, std::string &sysFsPciPath, int subDeviceId, int &maxGpuFrequency) {
maxGpuFrequency = 0;
std::string clockSysFsPath = sysFsPciPath + drm.getIoctlHelper()->getFileForMaxGpuFrequencyOfSubDevice(subDeviceId);
std::ifstream ifs(clockSysFsPath.c_str(), std::ifstream::in);
if (ifs.fail()) {
return -1;
}
ifs >> maxGpuFrequency;
ifs.close();
return 0;
}
int Drm::getMaxGpuFrequency(HardwareInfo &hwInfo, int &maxGpuFrequency) {
int ret = 0;
std::string sysFsPciPath = getSysFsPciPath();
auto tileCount = hwInfo.gtSystemInfo.MultiTileArchInfo.TileCount;
if (hwInfo.gtSystemInfo.MultiTileArchInfo.IsValid && tileCount > 0) {
for (auto tileId = 0; tileId < tileCount; tileId++) {
int maxGpuFreqOfSubDevice = 0;
ret |= getMaxGpuFrequencyOfSubDevice(*this, sysFsPciPath, tileId, maxGpuFreqOfSubDevice);
maxGpuFrequency = std::max(maxGpuFrequency, maxGpuFreqOfSubDevice);
}
if (ret == 0) {
return 0;
}
}
return getMaxGpuFrequencyOfDevice(*this, sysFsPciPath, maxGpuFrequency);
}
bool Drm::getDeviceMemoryMaxClockRateInMhz(uint32_t tileId, uint32_t &clkRate) {
const std::string relativefilePath = ioctlHelper->getFileForMaxMemoryFrequencyOfSubDevice(tileId);
std::string readString(64, '\0');
errno = 0;
if (readSysFsAsString(relativefilePath, readString) == false) {
return false;
}
char *endPtr = nullptr;
uint32_t retClkRate = static_cast<uint32_t>(std::strtoul(readString.data(), &endPtr, 10));
if ((endPtr == readString.data()) || (errno != 0)) {
return false;
}
clkRate = retClkRate;
return true;
}
bool Drm::getDeviceMemoryPhysicalSizeInBytes(uint32_t tileId, uint64_t &physicalSize) {
const std::string relativefilePath = "/gt/gt" + std::to_string(tileId) + "/addr_range";
std::string readString(64, '\0');
errno = 0;
if (readSysFsAsString(relativefilePath, readString) == false) {
return false;
}
char *endPtr = nullptr;
uint64_t retSize = static_cast<uint64_t>(std::strtoull(readString.data(), &endPtr, 16));
if ((endPtr == readString.data()) || (errno != 0)) {
return false;
}
physicalSize = retSize;
return true;
}
bool Drm::useVMBindImmediate() const {
bool useBindImmediate = isDirectSubmissionActive() || hasPageFaultSupport() || ioctlHelper->isImmediateVmBindRequired();
if (debugManager.flags.EnableImmediateVmBindExt.get() != -1) {
useBindImmediate = debugManager.flags.EnableImmediateVmBindExt.get();
}
return useBindImmediate;
}
void Drm::setupSystemInfo(HardwareInfo *hwInfo, SystemInfo *sysInfo) {
GT_SYSTEM_INFO *gtSysInfo = &hwInfo->gtSystemInfo;
gtSysInfo->MaxEuPerSubSlice = sysInfo->getMaxEuPerDualSubSlice();
if (!gtSysInfo->EUCount) {
gtSysInfo->EUCount = gtSysInfo->SubSliceCount * gtSysInfo->MaxEuPerSubSlice;
}
gtSysInfo->ThreadCount = gtSysInfo->EUCount * sysInfo->getNumThreadsPerEu();
gtSysInfo->MemoryType = sysInfo->getMemoryType();
gtSysInfo->MaxSlicesSupported = sysInfo->getMaxSlicesSupported();
gtSysInfo->MaxSubSlicesSupported = sysInfo->getMaxDualSubSlicesSupported();
gtSysInfo->MaxDualSubSlicesSupported = sysInfo->getMaxDualSubSlicesSupported();
gtSysInfo->CsrSizeInMb = sysInfo->getCsrSizeInMb();
gtSysInfo->SLMSizeInKb = sysInfo->getSlmSizePerDss();
if (!hwInfo->capabilityTable.slmSize) {
hwInfo->capabilityTable.slmSize = gtSysInfo->SLMSizeInKb;
}
}
void Drm::setupCacheInfo(const HardwareInfo &hwInfo) {
auto &gfxCoreHelper = rootDeviceEnvironment.getHelper<GfxCoreHelper>();
if (debugManager.flags.ClosEnabled.get() == 0 || gfxCoreHelper.getNumCacheRegions() == 0) {
this->cacheInfo.reset(new CacheInfo(*this, 0, 0, 0));
return;
}
const GT_SYSTEM_INFO *gtSysInfo = &hwInfo.gtSystemInfo;
constexpr uint16_t maxNumWays = 32;
constexpr uint16_t globalReservationLimit = 16;
constexpr uint16_t clientReservationLimit = 8;
constexpr uint16_t maxReservationNumWays = std::min(globalReservationLimit, clientReservationLimit);
const size_t totalCacheSize = gtSysInfo->L3CacheSizeInKb * MemoryConstants::kiloByte;
const size_t maxReservationCacheSize = (totalCacheSize * maxReservationNumWays) / maxNumWays;
const uint32_t maxReservationNumCacheRegions = gfxCoreHelper.getNumCacheRegions() - 1;
this->cacheInfo.reset(new CacheInfo(*this, maxReservationCacheSize, maxReservationNumCacheRegions, maxReservationNumWays));
}
void Drm::getPrelimVersion(std::string &prelimVersion) {
std::string sysFsPciPath = getSysFsPciPath();
std::string prelimVersionPath = sysFsPciPath + "/prelim_uapi_version";
std::ifstream ifs(prelimVersionPath.c_str(), std::ifstream::in);
if (ifs.fail()) {
prelimVersion = "";
} else {
ifs >> prelimVersion;
}
ifs.close();
}
int Drm::waitUserFence(uint32_t ctxId, uint64_t address, uint64_t value, ValueWidth dataWidth, int64_t timeout, uint16_t flags, bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) {
return ioctlHelper->waitUserFence(ctxId, address, value, static_cast<uint32_t>(dataWidth), timeout, flags, userInterrupt, externalInterruptId, allocForInterruptWait);
}
bool Drm::querySystemInfo() {
auto request = ioctlHelper->getDrmParamValue(DrmParam::queryHwconfigTable);
auto deviceBlobQuery = this->query<uint32_t>(request, 0);
if (deviceBlobQuery.empty()) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stdout, "%s", "INFO: System Info query failed!\n");
return false;
}
this->systemInfo.reset(new SystemInfo(deviceBlobQuery));
return true;
}
std::vector<uint64_t> Drm::getMemoryRegions() {
auto request = ioctlHelper->getDrmParamValue(DrmParam::queryMemoryRegions);
return this->query<uint64_t>(request, 0);
}
bool Drm::queryMemoryInfo() {
this->memoryInfo = ioctlHelper->createMemoryInfo();
return this->memoryInfo != nullptr;
}
bool Drm::queryEngineInfo(bool isSysmanEnabled) {
this->engineInfo = ioctlHelper->createEngineInfo(isSysmanEnabled);
if (this->engineInfo && (this->engineInfo->hasEngines() == false)) {
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Engine info size is equal to 0.\n");
}
return this->engineInfo != nullptr;
}
bool Drm::completionFenceSupport() {
std::call_once(checkCompletionFenceOnce, [this]() {
const bool vmBindAvailable = isVmBindAvailable();
bool support = ioctlHelper->completionFenceExtensionSupported(vmBindAvailable);
int32_t overrideCompletionFence = debugManager.flags.EnableDrmCompletionFence.get();
if (overrideCompletionFence != -1) {
support = !!overrideCompletionFence;
}
completionFenceSupported = support;
if (debugManager.flags.PrintCompletionFenceUsage.get()) {
std::cout << "Completion fence supported: " << completionFenceSupported << std::endl;
}
});
return completionFenceSupported;
}
void Drm::setupIoctlHelper(const PRODUCT_FAMILY productFamily) {
if (!this->ioctlHelper) {
std::string prelimVersion = "";
getPrelimVersion(prelimVersion);
this->ioctlHelper = IoctlHelper::getI915Helper(productFamily, prelimVersion, *this);
this->ioctlHelper->initialize();
}
}
bool Drm::queryTopology(const HardwareInfo &hwInfo, DrmQueryTopologyData &topologyData) {
auto result = this->ioctlHelper->getTopologyDataAndMap(hwInfo, topologyData, topologyMap);
return result;
}
void Drm::queryPageFaultSupport() {
const auto &productHelper = this->getRootDeviceEnvironment().getHelper<ProductHelper>();
if (!productHelper.isPageFaultSupported()) {
return;
}
if (const auto paramId = ioctlHelper->getHasPageFaultParamId(); paramId) {
int support = 0;
const auto ret = getParamIoctl(*paramId, &support);
pageFaultSupported = (0 == ret) && (support > 0);
}
}
bool Drm::hasPageFaultSupport() const {
if (debugManager.flags.EnableRecoverablePageFaults.get() != -1) {
return !!debugManager.flags.EnableRecoverablePageFaults.get();
}
return pageFaultSupported;
}
bool Drm::hasKmdMigrationSupport() const {
const auto &productHelper = this->getRootDeviceEnvironment().getHelper<ProductHelper>();
auto kmdMigrationSupported = hasPageFaultSupport() && productHelper.isKmdMigrationSupported();
if (debugManager.flags.UseKmdMigration.get() != -1) {
return !!debugManager.flags.UseKmdMigration.get();
}
return kmdMigrationSupported;
}
void Drm::configureScratchPagePolicy() {
if (debugManager.flags.DisableScratchPages.get() != -1) {
disableScratch = !!debugManager.flags.DisableScratchPages.get();
return;
}
const auto &productHelper = this->getRootDeviceEnvironment().getHelper<ProductHelper>();
disableScratch = (productHelper.isDisableScratchPagesSupported() &&
!rootDeviceEnvironment.executionEnvironment.isDebuggingEnabled());
}
void Drm::configureGpuFaultCheckThreshold() {
if (debugManager.flags.GpuFaultCheckThreshold.get() != -1) {
gpuFaultCheckThreshold = debugManager.flags.GpuFaultCheckThreshold.get();
}
}
unsigned int Drm::bindDrmContext(uint32_t drmContextId, uint32_t deviceIndex, aub_stream::EngineType engineType, bool engineInstancedDevice) {
auto engineInfo = this->engineInfo.get();
auto retVal = static_cast<unsigned int>(ioctlHelper->getDrmParamValue(DrmEngineMapper::engineNodeMap(engineType)));
if (!engineInfo) {
return retVal;
}
auto engine = engineInfo->getEngineInstance(deviceIndex, engineType);
if (!engine) {
return retVal;
}
bool useVirtualEnginesForCcs = !engineInstancedDevice;
if (debugManager.flags.UseDrmVirtualEnginesForCcs.get() != -1) {
useVirtualEnginesForCcs = !!debugManager.flags.UseDrmVirtualEnginesForCcs.get();
}
auto numberOfCCS = rootDeviceEnvironment.getHardwareInfo()->gtSystemInfo.CCSInfo.NumberOfCCSEnabled;
constexpr uint32_t maxEngines = 9u;
bool useVirtualEnginesForBcs = EngineHelpers::isBcsVirtualEngineEnabled(engineType);
auto numberOfBCS = rootDeviceEnvironment.getHardwareInfo()->featureTable.ftrBcsInfo.count();
if (debugManager.flags.LimitEngineCountForVirtualBcs.get() != -1) {
numberOfBCS = debugManager.flags.LimitEngineCountForVirtualBcs.get();
}
if (debugManager.flags.LimitEngineCountForVirtualCcs.get() != -1) {
numberOfCCS = debugManager.flags.LimitEngineCountForVirtualCcs.get();
}
uint32_t numEnginesInContext = 1;
ContextParamEngines<> contextEngines{};
ContextEnginesLoadBalance<maxEngines> balancer{};
ioctlHelper->insertEngineToContextParams(contextEngines, 0u, engine, deviceIndex, false);
bool setupVirtualEngines = false;
unsigned int engineCount = static_cast<unsigned int>(numberOfCCS);
if (useVirtualEnginesForCcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCompute) && numberOfCCS > 1u) {
numEnginesInContext = numberOfCCS + 1;
balancer.numSiblings = numberOfCCS;
setupVirtualEngines = true;
}
bool includeMainCopyEngineInGroup = false;
if (useVirtualEnginesForBcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCopy) && numberOfBCS > 1u) {
numEnginesInContext = static_cast<uint32_t>(numberOfBCS) + 1;
balancer.numSiblings = numberOfBCS;
setupVirtualEngines = true;
engineCount = static_cast<unsigned int>(rootDeviceEnvironment.getHardwareInfo()->featureTable.ftrBcsInfo.size());
if (EngineHelpers::getBcsIndex(engineType) == 0u) {
includeMainCopyEngineInGroup = true;
} else {
engineCount--;
balancer.numSiblings = numberOfBCS - 1;
numEnginesInContext = static_cast<uint32_t>(numberOfBCS);
}
}
if (setupVirtualEngines) {
balancer.base.name = ioctlHelper->getDrmParamValue(DrmParam::contextEnginesExtLoadBalance);
contextEngines.extensions = castToUint64(&balancer);
ioctlHelper->insertEngineToContextParams(contextEngines, 0u, nullptr, deviceIndex, true);
for (auto engineIndex = 0u; engineIndex < engineCount; engineIndex++) {
if (useVirtualEnginesForBcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCopy)) {
auto mappedBcsEngineType = static_cast<aub_stream::EngineType>(EngineHelpers::mapBcsIndexToEngineType(engineIndex, includeMainCopyEngineInGroup));
bool isBcsEnabled = rootDeviceEnvironment.getHardwareInfo()->featureTable.ftrBcsInfo.test(EngineHelpers::getBcsIndex(mappedBcsEngineType));
if (!isBcsEnabled) {
continue;
}
engine = engineInfo->getEngineInstance(deviceIndex, mappedBcsEngineType);
}
UNRECOVERABLE_IF(!engine);
if (useVirtualEnginesForCcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCompute)) {
engine = engineInfo->getEngineInstance(deviceIndex, static_cast<aub_stream::EngineType>(EngineHelpers::mapCcsIndexToEngineType(engineIndex)));
}
UNRECOVERABLE_IF(!engine);
balancer.engines[engineIndex] = {engine->engineClass, engine->engineInstance};
ioctlHelper->insertEngineToContextParams(contextEngines, engineIndex, engine, deviceIndex, true);
}
}
GemContextParam param{};
param.contextId = drmContextId;
param.size = static_cast<uint32_t>(ptrDiff(contextEngines.enginesData, &contextEngines) + sizeof(EngineClassInstance) * numEnginesInContext);
param.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamEngines);
param.value = castToUint64(&contextEngines);
auto ioctlValue = ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &param);
UNRECOVERABLE_IF(ioctlValue != 0);
retVal = static_cast<unsigned int>(ioctlHelper->getDrmParamValue(DrmParam::execDefault));
return retVal;
}
void Drm::waitForBind(uint32_t vmHandleId) {
if (pagingFence[vmHandleId] >= fenceVal[vmHandleId]) {
return;
}
auto lock = this->lockBindFenceMutex();
auto fenceAddress = castToUint64(&this->pagingFence[vmHandleId]);
auto fenceValue = this->fenceVal[vmHandleId];
lock.unlock();
waitUserFence(0u, fenceAddress, fenceValue, ValueWidth::u64, -1, ioctlHelper->getWaitUserFenceSoftFlag(), false, NEO::InterruptId::notUsed, nullptr);
}
bool Drm::isSetPairAvailable() {
if (debugManager.flags.EnableSetPair.get() == 1) {
std::call_once(checkSetPairOnce, [this]() {
int ret = ioctlHelper->isSetPairAvailable();
setPairAvailable = ret;
});
}
return setPairAvailable;
}
bool Drm::isChunkingAvailable() {
if (debugManager.flags.EnableBOChunking.get() != 0) {
std::call_once(checkChunkingOnce, [this]() {
int ret = ioctlHelper->isChunkingAvailable();
if (ret) {
if (debugManager.flags.EnableBOChunking.get() == -1) {
chunkingMode = chunkingModeDevice;
} else {
chunkingMode = debugManager.flags.EnableBOChunking.get();
if (!(hasKmdMigrationSupport())) {
chunkingMode &= (~(chunkingModeShared));
}
}
}
if (chunkingMode > 0) {
chunkingAvailable = true;
}
if (debugManager.flags.MinimalAllocationSizeForChunking.get() != -1) {
minimalChunkingSize = debugManager.flags.MinimalAllocationSizeForChunking.get();
}
printDebugString(debugManager.flags.PrintBOChunkingLogs.get(), stdout,
"Chunking available: %d; enabled for: shared allocations %d, device allocations %d; minimalChunkingSize: %zd\n",
chunkingAvailable,
(chunkingMode & chunkingModeShared),
(chunkingMode & chunkingModeDevice),
minimalChunkingSize);
});
}
return chunkingAvailable;
}
bool Drm::isVmBindAvailable() {
std::call_once(checkBindOnce, [this]() {
bindAvailable = ioctlHelper->isVmBindAvailable();
Drm::overrideBindSupport(bindAvailable);
queryAndSetVmBindPatIndexProgrammingSupport();
});
return bindAvailable;
}
uint64_t Drm::getPatIndex(Gmm *gmm, AllocationType allocationType, CacheRegion cacheRegion, CachePolicy cachePolicy, bool closEnabled, bool isSystemMemory) const {
if ((debugManager.flags.OverridePatIndexForSystemMemory.get() != -1) && isSystemMemory) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForSystemMemory.get());
}
if ((debugManager.flags.OverridePatIndexForDeviceMemory.get() != -1) && !isSystemMemory) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForDeviceMemory.get());
}
if (debugManager.flags.OverridePatIndex.get() != -1) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndex.get());
}
auto &productHelper = rootDeviceEnvironment.getProductHelper();
GMM_RESOURCE_USAGE_TYPE usageType = CacheSettingsHelper::getGmmUsageType(allocationType, false, productHelper);
auto isUncachedType = CacheSettingsHelper::isUncachedType(usageType);
if (isUncachedType && debugManager.flags.OverridePatIndexForUncachedTypes.get() != -1) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForUncachedTypes.get());
}
if (!isUncachedType && debugManager.flags.OverridePatIndexForCachedTypes.get() != -1) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForCachedTypes.get());
}
if (!this->vmBindPatIndexProgrammingSupported) {
return CommonConstants::unsupportedPatIndex;
}
auto &gfxCoreHelper = rootDeviceEnvironment.getHelper<GfxCoreHelper>();
GMM_RESOURCE_INFO *resourceInfo = nullptr;
bool cachable = !CacheSettingsHelper::isUncachedType(usageType);
bool compressed = false;
if (gmm) {
resourceInfo = gmm->gmmResourceInfo->peekGmmResourceInfo();
usageType = gmm->resourceParams.Usage;
compressed = gmm->isCompressionEnabled();
cachable = gmm->gmmResourceInfo->getResourceFlags()->Info.Cacheable;
}
uint64_t patIndex = rootDeviceEnvironment.getGmmClientContext()->cachePolicyGetPATIndex(resourceInfo, usageType, compressed, cachable);
patIndex = productHelper.overridePatIndex(isUncachedType, patIndex, allocationType);
UNRECOVERABLE_IF(patIndex == static_cast<uint64_t>(GMM_PAT_ERROR));
if (debugManager.flags.ClosEnabled.get() != -1) {
closEnabled = !!debugManager.flags.ClosEnabled.get();
}
if (closEnabled) {
patIndex = gfxCoreHelper.getPatIndex(cacheRegion, cachePolicy);
}
return patIndex;
}
void programUserFence(Drm *drm, OsContext *osContext, BufferObject *bo, VmBindExtUserFenceT &vmBindExtUserFence, uint32_t vmHandleId, uint64_t nextExtension) {
auto ioctlHelper = drm->getIoctlHelper();
uint64_t address = 0;
uint64_t value = 0;
if (drm->isPerContextVMRequired()) {
auto osContextLinux = static_cast<OsContextLinux *>(osContext);
address = castToUint64(osContextLinux->getFenceAddr(vmHandleId));
value = osContextLinux->getNextFenceVal(vmHandleId);
} else {
address = castToUint64(drm->getFenceAddr(vmHandleId));
value = drm->getNextFenceVal(vmHandleId);
}
ioctlHelper->fillVmBindExtUserFence(vmBindExtUserFence, address, value, nextExtension);
}
int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleId, BufferObject *bo, bool bind) {
auto vmId = drm->getVirtualMemoryAddressSpace(vmHandleId);
auto ioctlHelper = drm->getIoctlHelper();
uint64_t flags = 0u;
if (drm->isPerContextVMRequired()) {
auto osContextLinux = static_cast<const OsContextLinux *>(osContext);
UNRECOVERABLE_IF(osContextLinux->getDrmVmIds().size() <= vmHandleId);
vmId = osContextLinux->getDrmVmIds()[vmHandleId];
}
std::unique_ptr<uint8_t[]> extensions;
if (bind) {
bool allowUUIDsForDebug = !osContext->isInternalEngine() && !EngineHelpers::isBcs(osContext->getEngineType());
if (bo->getBindExtHandles().size() > 0 && allowUUIDsForDebug) {
extensions = ioctlHelper->prepareVmBindExt(bo->getBindExtHandles());
}
bool bindCapture = bo->isMarkedForCapture();
bool bindImmediate = bo->isImmediateBindingRequired();
bool bindMakeResident = false;
bool readOnlyResource = bo->isReadOnlyGpuResource();
if (drm->useVMBindImmediate()) {
bindMakeResident = bo->isExplicitResidencyRequired();
bindImmediate = true;
}
bool bindLock = bo->isExplicitLockedMemoryRequired();
flags |= ioctlHelper->getFlagsForVmBind(bindCapture, bindImmediate, bindMakeResident, bindLock, readOnlyResource);
}
auto &bindAddresses = bo->getColourAddresses();
auto bindIterations = bindAddresses.size();
if (bindIterations == 0) {
bindIterations = 1;
}
int ret = 0;
for (size_t i = 0; i < bindIterations; i++) {
VmBindParams vmBind{};
vmBind.vmId = static_cast<uint32_t>(vmId);
vmBind.flags = flags;
vmBind.handle = bo->peekHandle();
vmBind.length = bo->peekSize();
vmBind.offset = 0;
vmBind.start = bo->peekAddress();
vmBind.userptr = bo->getUserptr();
if (bo->getColourWithBind()) {
vmBind.length = bo->getColourChunk();
vmBind.offset = bo->getColourChunk() * i;
vmBind.start = bindAddresses[i];
}
VmBindExtSetPatT vmBindExtSetPat{};
if (drm->isVmBindPatIndexProgrammingSupported()) {
UNRECOVERABLE_IF(bo->peekPatIndex() == CommonConstants::unsupportedPatIndex);
if (ioctlHelper->isVmBindPatIndexExtSupported()) {
ioctlHelper->fillVmBindExtSetPat(vmBindExtSetPat, bo->peekPatIndex(), castToUint64(extensions.get()));
vmBind.extensions = castToUint64(vmBindExtSetPat);
} else {
vmBind.extensions = castToUint64(extensions.get());
}
vmBind.patIndex = bo->peekPatIndex();
} else {
vmBind.extensions = castToUint64(extensions.get());
}
std::unique_lock<std::mutex> lock;
VmBindExtUserFenceT vmBindExtUserFence{};
bool incrementFenceValue = false;
if (ioctlHelper->isWaitBeforeBindRequired(bind)) {
if (drm->useVMBindImmediate()) {
lock = drm->lockBindFenceMutex();
if (!drm->hasPageFaultSupport() || bo->isExplicitResidencyRequired()) {
auto nextExtension = vmBind.extensions;
incrementFenceValue = true;
programUserFence(drm, osContext, bo, vmBindExtUserFence, vmHandleId, nextExtension);
ioctlHelper->setVmBindUserFence(vmBind, vmBindExtUserFence);
}
}
}
if (bind) {
ret = ioctlHelper->vmBind(vmBind);
if (ret) {
break;
}
drm->setNewResourceBoundToVM(bo, vmHandleId);
} else {
vmBind.handle = 0u;
ret = ioctlHelper->vmUnbind(vmBind);
if (ret) {
break;
}
}
bool waitOnUserFenceAfterBindAndUnbind = false;
if (debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get() != -1) {
waitOnUserFenceAfterBindAndUnbind = !!debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get();
}
if (ioctlHelper->isWaitBeforeBindRequired(bind) && waitOnUserFenceAfterBindAndUnbind && drm->useVMBindImmediate()) {
auto osContextLinux = static_cast<OsContextLinux *>(osContext);
osContextLinux->waitForPagingFence();
}
if (incrementFenceValue) {
if (drm->isPerContextVMRequired()) {
auto osContextLinux = static_cast<OsContextLinux *>(osContext);
osContextLinux->incFenceVal(vmHandleId);
} else {
drm->incFenceVal(vmHandleId);
}
}
}
return ret;
}
int Drm::bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) {
auto ret = changeBufferObjectBinding(this, osContext, vmHandleId, bo, true);
if (ret != 0) {
static_cast<DrmMemoryOperationsHandlerBind *>(this->rootDeviceEnvironment.memoryOperationsInterface.get())->evictUnusedAllocations(false, false);
ret = changeBufferObjectBinding(this, osContext, vmHandleId, bo, true);
}
return ret;
}
int Drm::unbindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) {
return changeBufferObjectBinding(this, osContext, vmHandleId, bo, false);
}
int Drm::createDrmVirtualMemory(uint32_t &drmVmId) {
GemVmControl ctl{};
std::optional<MemoryClassInstance> regionInstanceClass;
uint32_t memoryBank = 1 << drmVmId;
auto hwInfo = this->getRootDeviceEnvironment().getHardwareInfo();
auto memInfo = this->getMemoryInfo();
if (debugManager.flags.UseTileMemoryBankInVirtualMemoryCreation.get() != 0) {
if (memInfo && rootDeviceEnvironment.getHelper<GfxCoreHelper>().getEnableLocalMemory(*hwInfo)) {
regionInstanceClass = memInfo->getMemoryRegionClassAndInstance(memoryBank, *this->rootDeviceEnvironment.getHardwareInfo());
}
}
auto vmControlExtRegion = ioctlHelper->createVmControlExtRegion(regionInstanceClass);
if (vmControlExtRegion) {
ctl.extensions = castToUint64(vmControlExtRegion.get());
}
bool useVmBind = isVmBindAvailable();
bool enablePageFault = hasPageFaultSupport() && useVmBind;
ctl.flags = ioctlHelper->getFlagsForVmCreate(checkToDisableScratchPage(), enablePageFault, useVmBind);
auto ret = ioctlHelper->ioctl(DrmIoctl::gemVmCreate, &ctl);
if (ret == 0) {
drmVmId = ctl.vmId;
if (ctl.vmId == 0) {
// 0 is reserved for invalid/unassigned ppgtt
return -1;
}
} else {
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr,
"INFO: Cannot create Virtual Memory at memory bank 0x%x info present %d return code %d\n",
memoryBank, memoryInfo != nullptr, ret);
}
return ret;
}
PhysicalDevicePciSpeedInfo Drm::getPciSpeedInfo() const {
PhysicalDevicePciSpeedInfo pciSpeedInfo = {};
std::string pathPrefix{};
bool isIntegratedDevice = rootDeviceEnvironment.getHardwareInfo()->capabilityTable.isIntegratedDevice;
// If integrated device, read properties from the specific device path.
// If discrete device, read properties from the root path of the pci device.
if (isIntegratedDevice) {
auto devicePath = NEO::getPciLinkPath(getFileDescriptor());
if (!devicePath.has_value()) {
return pciSpeedInfo;
}
pathPrefix = "/sys/class/drm/" + devicePath.value() + "/device/";
} else {
auto rootPath = NEO::getPciRootPath(getFileDescriptor());
if (!rootPath.has_value()) {
return pciSpeedInfo;
}
pathPrefix += "/sys/devices" + rootPath.value();
}
std::array<char, 32> readString = {'\0'};
errno = 0;
auto readFile = [](const std::string fileName, const std::string_view pathPrefix, std::array<char, 32> &readString) {
std::ostringstream linkWidthStream{};
linkWidthStream << pathPrefix << fileName;
int fd = NEO::SysCalls::open(linkWidthStream.str().c_str(), O_RDONLY);
if (fd < 0) {
return false;
}
ssize_t bytesRead = NEO::SysCalls::pread(fd, readString.data(), readString.size() - 1, 0);
NEO::SysCalls::close(fd);
if (bytesRead <= 0) {
return false;
}
std::replace(readString.begin(), readString.end(), '\n', '\0');
return true;
};
// read max link width
if (readFile("/max_link_width", pathPrefix, readString) != true) {
return pciSpeedInfo;
}
char *endPtr = nullptr;
uint32_t linkWidth = static_cast<uint32_t>(std::strtoul(readString.data(), &endPtr, 10));
if ((endPtr == readString.data()) || (errno != 0)) {
return pciSpeedInfo;
}
pciSpeedInfo.width = linkWidth;
// read max link speed
if (readFile("/max_link_speed", pathPrefix, readString) != true) {
return pciSpeedInfo;
}
endPtr = nullptr;
const auto maxSpeed = strtod(readString.data(), &endPtr);
if ((endPtr == readString.data()) || (errno != 0)) {
return pciSpeedInfo;
}
double gen3EncodingLossFactor = 128.0 / 130.0;
std::map<double, std::pair<int32_t, double>> maxSpeedToGenAndEncodingLossMapping{
//{max link speed, {pci generation, encoding loss factor}}
{2.5, {1, 0.2}},
{5.0, {2, 0.2}},
{8.0, {3, gen3EncodingLossFactor}},
{16.0, {4, gen3EncodingLossFactor}},
{32.0, {5, gen3EncodingLossFactor}}};
if (maxSpeedToGenAndEncodingLossMapping.find(maxSpeed) == maxSpeedToGenAndEncodingLossMapping.end()) {
return pciSpeedInfo;
}
pciSpeedInfo.genVersion = maxSpeedToGenAndEncodingLossMapping[maxSpeed].first;
constexpr double gigaBitsPerSecondToBytesPerSecondMultiplier = 125000000;
const auto maxSpeedWithEncodingLoss = maxSpeed * gigaBitsPerSecondToBytesPerSecondMultiplier * maxSpeedToGenAndEncodingLossMapping[maxSpeed].second;
pciSpeedInfo.maxBandwidth = static_cast<int64_t>(maxSpeedWithEncodingLoss * pciSpeedInfo.width);
return pciSpeedInfo;
}
int Drm::waitOnUserFences(OsContextLinux &osContext, uint64_t address, uint64_t value, uint32_t numActiveTiles, int64_t timeout, uint32_t postSyncOffset, bool userInterrupt,
uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) {
int ret = waitOnUserFencesImpl(static_cast<const OsContextLinux &>(osContext), address, value, numActiveTiles,
timeout, postSyncOffset, userInterrupt, externalInterruptId, allocForInterruptWait);
if (ret != 0 && getErrno() == EIO && checkGpuPageFaultRequired()) {
checkResetStatus(osContext);
}
return ret;
}
int Drm::waitOnUserFencesImpl(const OsContextLinux &osContext, uint64_t address, uint64_t value, uint32_t numActiveTiles, int64_t timeout, uint32_t postSyncOffset, bool userInterrupt,
uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) {
auto &drmContextIds = osContext.getDrmContextIds();
UNRECOVERABLE_IF(numActiveTiles > drmContextIds.size());
auto completionFenceCpuAddress = address;
const auto selectedTimeout = osContext.isHangDetected() ? 1 : timeout;
for (auto drmIterator = 0u; drmIterator < numActiveTiles; drmIterator++) {
if (*reinterpret_cast<uint32_t *>(completionFenceCpuAddress) < value) {
static constexpr uint16_t flags = 0;
int retVal = waitUserFence(drmContextIds[drmIterator], completionFenceCpuAddress, value, Drm::ValueWidth::u64, selectedTimeout, flags, userInterrupt, externalInterruptId, allocForInterruptWait);
if (debugManager.flags.PrintCompletionFenceUsage.get()) {
std::cout << "Completion fence waited."
<< " Status: " << retVal
<< ", CPU address: " << std::hex << completionFenceCpuAddress << std::dec
<< ", current value: " << *reinterpret_cast<uint32_t *>(completionFenceCpuAddress)
<< ", wait value: " << value << std::endl;
}
if (retVal != 0) {
return retVal;
}
} else if (debugManager.flags.PrintCompletionFenceUsage.get()) {
std::cout << "Completion fence already completed."
<< " CPU address: " << std::hex << completionFenceCpuAddress << std::dec
<< ", current value: " << *reinterpret_cast<uint32_t *>(completionFenceCpuAddress)
<< ", wait value: " << value << std::endl;
}
if (externalInterruptId != NEO::InterruptId::notUsed) {
break;
}
completionFenceCpuAddress = ptrOffset(completionFenceCpuAddress, postSyncOffset);
}
return 0;
}
const HardwareInfo *Drm::getHardwareInfo() const { return rootDeviceEnvironment.getHardwareInfo(); }
uint64_t Drm::alignUpGttSize(uint64_t inputGttSize) {
constexpr uint64_t gttSize47bit = (1ull << 47);
constexpr uint64_t gttSize48bit = (1ull << 48);
if (inputGttSize > gttSize47bit && inputGttSize < gttSize48bit) {
return gttSize48bit;
}
return inputGttSize;
}
template std::vector<uint16_t> Drm::query<uint16_t>(uint32_t queryId, uint32_t queryItemFlags);
template std::vector<uint32_t> Drm::query<uint32_t>(uint32_t queryId, uint32_t queryItemFlags);
template std::vector<uint64_t> Drm::query<uint64_t>(uint32_t queryId, uint32_t queryItemFlags);
} // namespace NEO
Reference in New Issue View Git Blame Copy Permalink