Media
/
compute-runtime
mirror of https://github.com/intel/compute-runtime.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
compute-runtime/shared/source/os_interface/linux/drm_neo.cpp

1664 lines
64 KiB
C++
Raw Blame History

/*
* Copyright (C) 2018-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/os_interface/linux/drm_neo.h"
#include "shared/source/command_stream/submission_status.h"
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/execution_environment/execution_environment.h"
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/gmm_helper/cache_settings_helper.h"
#include "shared/source/gmm_helper/client_context/gmm_client_context.h"
#include "shared/source/gmm_helper/gmm.h"
#include "shared/source/gmm_helper/resource_info.h"
#include "shared/source/helpers/aligned_memory.h"
#include "shared/source/helpers/basic_math.h"
#include "shared/source/helpers/constants.h"
#include "shared/source/helpers/debug_helpers.h"
#include "shared/source/helpers/gfx_core_helper.h"
#include "shared/source/helpers/gpu_page_fault_helper.h"
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/os_interface/driver_info.h"
#include "shared/source/os_interface/linux/cache_info.h"
#include "shared/source/os_interface/linux/drm_buffer_object.h"
#include "shared/source/os_interface/linux/drm_engine_mapper.h"
#include "shared/source/os_interface/linux/drm_gem_close_worker.h"
#include "shared/source/os_interface/linux/drm_memory_manager.h"
#include "shared/source/os_interface/linux/drm_memory_operations_handler_bind.h"
#include "shared/source/os_interface/linux/drm_wrappers.h"
#include "shared/source/os_interface/linux/engine_info.h"
#include "shared/source/os_interface/linux/hw_device_id.h"
#include "shared/source/os_interface/linux/ioctl_helper.h"
#include "shared/source/os_interface/linux/memory_info.h"
#include "shared/source/os_interface/linux/os_context_linux.h"
#include "shared/source/os_interface/linux/os_inc.h"
#include "shared/source/os_interface/linux/pci_path.h"
#include "shared/source/os_interface/linux/sys_calls.h"
#include "shared/source/os_interface/linux/system_info.h"
#include "shared/source/os_interface/os_environment.h"
#include "shared/source/os_interface/os_interface.h"
#include "shared/source/os_interface/product_helper.h"
#include "shared/source/utilities/api_intercept.h"
#include "shared/source/utilities/directory.h"
#include "shared/source/utilities/io_functions.h"
#include <cstdio>
#include <cstring>
#include <fcntl.h>
#include <fstream>
#include <map>
#include <sstream>
namespace NEO {
Drm::Drm(std::unique_ptr<HwDeviceIdDrm> &&hwDeviceIdIn, RootDeviceEnvironment &rootDeviceEnvironment)
: DriverModel(DriverModelType::drm),
hwDeviceId(std::move(hwDeviceIdIn)), rootDeviceEnvironment(rootDeviceEnvironment) {
pagingFence.fill(0u);
fenceVal.fill(0u);
}
SubmissionStatus Drm::getSubmissionStatusFromReturnCode(int32_t retCode) {
switch (retCode) {
case 0:
return SubmissionStatus::success;
case EWOULDBLOCK:
case ENOMEM:
case ENOSPC:
return SubmissionStatus::outOfHostMemory;
case ENXIO:
return SubmissionStatus::outOfMemory;
default:
return SubmissionStatus::failed;
}
}
void Drm::queryAndSetVmBindPatIndexProgrammingSupport() {
auto &productHelper = rootDeviceEnvironment.getHelper<ProductHelper>();
this->vmBindPatIndexProgrammingSupported = productHelper.isVmBindPatIndexProgrammingSupported();
}
int Drm::ioctl(DrmIoctl request, void *arg) {
auto requestValue = getIoctlRequestValue(request, ioctlHelper.get());
int ret;
int returnedErrno = 0;
SYSTEM_ENTER();
do {
auto measureTime = debugManager.flags.PrintKmdTimes.get();
std::chrono::steady_clock::time_point start;
std::chrono::steady_clock::time_point end;
auto printIoctl = debugManager.flags.PrintIoctlEntries.get();
if (printIoctl) {
printf("IOCTL %s called\n", getIoctlString(request, ioctlHelper.get()).c_str());
}
if (measureTime) {
start = std::chrono::steady_clock::now();
}
ret = SysCalls::ioctl(getFileDescriptor(), requestValue, arg);
if (ret != 0) {
returnedErrno = getErrno();
}
if (measureTime) {
end = std::chrono::steady_clock::now();
long long elapsedTime = std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count();
IoctlStatisticsEntry ioctlData{};
auto ioctlDataIt = this->ioctlStatistics.find(request);
if (ioctlDataIt != this->ioctlStatistics.end()) {
ioctlData = ioctlDataIt->second;
}
ioctlData.totalTime += elapsedTime;
ioctlData.count++;
ioctlData.minTime = std::min(ioctlData.minTime, elapsedTime);
ioctlData.maxTime = std::max(ioctlData.maxTime, elapsedTime);
this->ioctlStatistics[request] = ioctlData;
}
if (printIoctl) {
if (ret == 0) {
printf("IOCTL %s returns %d\n",
getIoctlString(request, ioctlHelper.get()).c_str(), ret);
} else {
printf("IOCTL %s returns %d, errno %d(%s)\n",
getIoctlString(request, ioctlHelper.get()).c_str(), ret, returnedErrno, strerror(returnedErrno));
}
}
} while (ret == -1 && checkIfIoctlReinvokeRequired(returnedErrno, request, ioctlHelper.get()));
SYSTEM_LEAVE(request);
return ret;
}
int Drm::getParamIoctl(DrmParam param, int *dstValue) {
GetParam getParam{};
getParam.param = getDrmParamValue(param, ioctlHelper.get());
getParam.value = dstValue;
int retVal = ioctlHelper ? ioctlHelper->ioctl(DrmIoctl::getparam, &getParam) : ioctl(DrmIoctl::getparam, &getParam);
if (debugManager.flags.PrintIoctlEntries.get()) {
printf("DRM_IOCTL_I915_GETPARAM: param: %s, output value: %d, retCode:% d\n",
getDrmParamString(param, ioctlHelper.get()).c_str(),
*getParam.value,
retVal);
}
return retVal;
}
bool Drm::queryI915DeviceIdAndRevision() {
HardwareInfo *hwInfo = rootDeviceEnvironment.getMutableHardwareInfo();
int deviceId = hwInfo->platform.usDeviceID;
int revisionId = hwInfo->platform.usRevId;
auto ret = getParamIoctl(DrmParam::paramChipsetId, &deviceId);
if (ret != 0) {
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Cannot query device ID parameter!\n");
return false;
}
ret = getParamIoctl(DrmParam::paramRevision, &revisionId);
if (ret != 0) {
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Cannot query device Rev ID parameter!\n");
return false;
}
hwInfo->platform.usDeviceID = deviceId;
hwInfo->platform.usRevId = revisionId;
return true;
}
int Drm::enableTurboBoost() {
GemContextParam contextParam = {};
contextParam.param = contextPrivateParamBoost;
contextParam.value = 1;
return ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
}
int Drm::getEnabledPooledEu(int &enabled) {
return getParamIoctl(DrmParam::paramHasPooledEu, &enabled);
}
std::string Drm::getSysFsPciPath() {
std::string path = std::string(Os::sysFsPciPathPrefix) + hwDeviceId->getPciPath() + "/drm";
std::string expectedFilePrefix = path + "/card";
auto files = Directory::getFiles(path.c_str());
for (auto &file : files) {
if (file.find(expectedFilePrefix.c_str()) != std::string::npos) {
return file;
}
}
return {};
}
bool Drm::readSysFsAsString(const std::string &relativeFilePath, std::string &readString) {
auto devicePath = getSysFsPciPath();
if (devicePath.empty()) {
return false;
}
const std::string fileName = devicePath + relativeFilePath;
int fd = SysCalls::open(fileName.c_str(), O_RDONLY);
if (fd < 0) {
return false;
}
ssize_t bytesRead = SysCalls::pread(fd, readString.data(), readString.size() - 1, 0);
NEO::SysCalls::close(fd);
if (bytesRead <= 0) {
return false;
}
std::replace(readString.begin(), readString.end(), '\n', '\0');
return true;
}
int Drm::queryGttSize(uint64_t &gttSizeOutput, bool alignUpToFullRange) {
GemContextParam contextParam = {0};
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamGttSize);
int ret = ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &contextParam);
if (ret == 0) {
if (alignUpToFullRange) {
gttSizeOutput = Drm::alignUpGttSize(contextParam.value);
} else {
gttSizeOutput = contextParam.value;
}
}
return ret;
}
bool Drm::isGpuHangDetected(OsContext &osContext) {
bool ret = checkResetStatus(osContext);
auto threshold = getGpuFaultCheckThreshold();
if (checkGpuPageFaultRequired()) {
if (gpuFaultCheckCounter >= threshold) {
auto memoryManager = static_cast<DrmMemoryManager *>(this->rootDeviceEnvironment.executionEnvironment.memoryManager.get());
memoryManager->checkUnexpectedGpuPageFault();
gpuFaultCheckCounter = 0;
return false;
}
gpuFaultCheckCounter++;
}
return ret;
}
bool Drm::checkResetStatus(OsContext &osContext) {
const auto osContextLinux = static_cast<OsContextLinux *>(&osContext);
const auto &drmContextIds = osContextLinux->getDrmContextIds();
for (const auto drmContextId : drmContextIds) {
ResetStats resetStats{};
resetStats.contextId = drmContextId;
ResetStatsFault fault{};
uint32_t status = 0;
const auto retVal{ioctlHelper->getResetStats(resetStats, &status, &fault)};
UNRECOVERABLE_IF(retVal != 0);
if (checkToDisableScratchPage() && ioctlHelper->validPageFault(fault.flags)) {
bool banned = ((status & ioctlHelper->getStatusForResetStats(true)) != 0);
IoFunctions::fprintf(stderr, "FATAL: Unexpected page fault from GPU at 0x%llx, ctx_id: %u (%s) type: %d (%s), level: %d (%s), access: %d (%s), banned: %d, aborting.\n",
fault.addr,
resetStats.contextId,
EngineHelpers::engineTypeToString(osContext.getEngineType()).c_str(),
fault.type, GpuPageFaultHelpers::faultTypeToString(static_cast<FaultType>(fault.type)).c_str(),
fault.level, GpuPageFaultHelpers::faultLevelToString(static_cast<FaultLevel>(fault.level)).c_str(),
fault.access, GpuPageFaultHelpers::faultAccessToString(static_cast<FaultAccess>(fault.access)).c_str(),
banned);
IoFunctions::fprintf(stdout, "FATAL: Unexpected page fault from GPU at 0x%llx, ctx_id: %u (%s) type: %d (%s), level: %d (%s), access: %d (%s), banned: %d, aborting.\n",
fault.addr,
resetStats.contextId,
EngineHelpers::engineTypeToString(osContext.getEngineType()).c_str(),
fault.type, GpuPageFaultHelpers::faultTypeToString(static_cast<FaultType>(fault.type)).c_str(),
fault.level, GpuPageFaultHelpers::faultLevelToString(static_cast<FaultLevel>(fault.level)).c_str(),
fault.access, GpuPageFaultHelpers::faultAccessToString(static_cast<FaultAccess>(fault.access)).c_str(),
banned);
UNRECOVERABLE_IF(true);
}
if (resetStats.batchActive > 0 || resetStats.batchPending > 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "ERROR: GPU HANG detected!\n");
osContextLinux->setHangDetected();
return true;
}
}
return false;
}
void Drm::checkPreemptionSupport() {
preemptionSupported = ioctlHelper->isPreemptionSupported();
}
void Drm::checkQueueSliceSupport() {
sliceCountChangeSupported = getQueueSliceCount(&sseu) == 0 ? true : false;
}
void Drm::setLowPriorityContextParam(uint32_t drmContextId) {
GemContextParam gcp = {};
gcp.contextId = drmContextId;
gcp.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamPriority);
gcp.value = -1023;
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &gcp);
UNRECOVERABLE_IF(retVal != 0);
}
int Drm::getQueueSliceCount(GemContextParamSseu *sseu) {
GemContextParam contextParam = {};
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamSseu);
sseu->engine.engineClass = ioctlHelper->getDrmParamValue(DrmParam::engineClassRender);
sseu->engine.engineInstance = ioctlHelper->getDrmParamValue(DrmParam::execDefault);
contextParam.value = reinterpret_cast<uint64_t>(sseu);
contextParam.size = sizeof(struct GemContextParamSseu);
return ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &contextParam);
}
uint64_t Drm::getSliceMask(uint64_t sliceCount) {
return maxNBitValue(sliceCount);
}
bool Drm::setQueueSliceCount(uint64_t sliceCount) {
if (sliceCountChangeSupported) {
GemContextParam contextParam = {};
sseu.sliceMask = getSliceMask(sliceCount);
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamSseu);
contextParam.contextId = 0;
contextParam.value = reinterpret_cast<uint64_t>(&sseu);
contextParam.size = sizeof(struct GemContextParamSseu);
int retVal = ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
if (retVal == 0) {
return true;
}
}
return false;
}
void Drm::checkNonPersistentContextsSupport() {
GemContextParam contextParam = {};
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamPersistence);
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &contextParam);
if (retVal == 0 && contextParam.value == 1) {
nonPersistentContextsSupported = true;
} else {
nonPersistentContextsSupported = false;
}
}
void Drm::setNonPersistentContext(uint32_t drmContextId) {
GemContextParam contextParam = {};
contextParam.contextId = drmContextId;
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamPersistence);
ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
}
void Drm::setUnrecoverableContext(uint32_t drmContextId) {
GemContextParam contextParam = {};
contextParam.contextId = drmContextId;
contextParam.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamRecoverable);
contextParam.value = 0;
contextParam.size = 0;
ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &contextParam);
}
int Drm::createDrmContext(uint32_t drmVmId, bool isDirectSubmissionRequested, bool isCooperativeContextRequested) {
GemContextCreateExt gcc{};
if (debugManager.flags.DirectSubmissionDrmContext.get() != -1) {
isDirectSubmissionRequested = debugManager.flags.DirectSubmissionDrmContext.get();
}
if (isDirectSubmissionRequested) {
gcc.flags |= ioctlHelper->getDirectSubmissionFlag();
}
GemContextCreateExtSetParam extSetparam = {};
if (drmVmId > 0) {
extSetparam.base.name = ioctlHelper->getDrmParamValue(DrmParam::contextCreateExtSetparam);
extSetparam.param.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamVm);
extSetparam.param.value = drmVmId;
gcc.extensions = reinterpret_cast<uint64_t>(&extSetparam);
gcc.flags |= ioctlHelper->getDrmParamValue(DrmParam::contextCreateFlagsUseExtensions);
}
if (debugManager.flags.CreateContextWithAccessCounters.get() > 0) {
return ioctlHelper->createContextWithAccessCounters(gcc);
}
if (debugManager.flags.ForceRunAloneContext.get() != -1) {
isCooperativeContextRequested = debugManager.flags.ForceRunAloneContext.get();
}
if (isCooperativeContextRequested) {
return ioctlHelper->createCooperativeContext(gcc);
}
auto ioctlResult = ioctlHelper->ioctl(DrmIoctl::gemContextCreateExt, &gcc);
if (ioctlResult < 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "WARNING: GemContextCreateExt ioctl failed. Not exposing this root device\n");
return ioctlResult;
}
return gcc.contextId;
}
void Drm::destroyDrmContext(uint32_t drmContextId) {
GemContextDestroy destroy{};
destroy.contextId = drmContextId;
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextDestroy, &destroy);
UNRECOVERABLE_IF((retVal != 0) && (errno != ENODEV));
}
void Drm::destroyDrmVirtualMemory(uint32_t drmVmId) {
GemVmControl ctl = {};
ctl.vmId = drmVmId;
auto ret = ioctlHelper->ioctl(DrmIoctl::gemVmDestroy, &ctl);
UNRECOVERABLE_IF((ret != 0) && (errno != ENODEV));
}
int Drm::queryVmId(uint32_t drmContextId, uint32_t &vmId) {
GemContextParam param{};
param.contextId = drmContextId;
param.value = 0;
param.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamVm);
auto retVal = ioctlHelper->ioctl(DrmIoctl::gemContextGetparam, &param);
vmId = static_cast<uint32_t>(param.value);
return retVal;
}
std::unique_lock<std::mutex> Drm::lockBindFenceMutex() {
return std::unique_lock<std::mutex>(this->bindFenceMutex);
}
int Drm::getEuTotal(int &euTotal) {
return getParamIoctl(DrmParam::paramEuTotal, &euTotal);
}
int Drm::getSubsliceTotal(int &subsliceTotal) {
return getParamIoctl(DrmParam::paramSubsliceTotal, &subsliceTotal);
}
int Drm::getMinEuInPool(int &minEUinPool) {
return getParamIoctl(DrmParam::paramMinEuInPool, &minEUinPool);
}
int Drm::getErrno() {
return errno;
}
int Drm::setupHardwareInfo(const DeviceDescriptor *device, bool setupFeatureTableAndWorkaroundTable) {
const auto usDeviceIdOverride = rootDeviceEnvironment.getHardwareInfo()->platform.usDeviceID;
const auto usRevIdOverride = rootDeviceEnvironment.getHardwareInfo()->platform.usRevId;
// reset hwInfo and apply overrides
rootDeviceEnvironment.setHwInfo(device->pHwInfo);
HardwareInfo *hwInfo = rootDeviceEnvironment.getMutableHardwareInfo();
hwInfo->platform.usDeviceID = usDeviceIdOverride;
hwInfo->platform.usRevId = usRevIdOverride;
rootDeviceEnvironment.initProductHelper();
rootDeviceEnvironment.initGfxCoreHelper();
rootDeviceEnvironment.initApiGfxCoreHelper();
rootDeviceEnvironment.initCompilerProductHelper();
rootDeviceEnvironment.initAilConfigurationHelper();
auto result = rootDeviceEnvironment.initAilConfiguration();
if (false == result) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: AIL creation failed!\n");
return -1;
}
const auto productFamily = hwInfo->platform.eProductFamily;
setupIoctlHelper(productFamily);
ioctlHelper->setupIpVersion();
rootDeviceEnvironment.initReleaseHelper();
DrmQueryTopologyData topologyData = {};
bool status = queryTopology(*hwInfo, topologyData);
if (!status) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "WARNING: Topology query failed!\n");
auto ret = getEuTotal(topologyData.euCount);
if (ret != 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Cannot query EU total parameter!\n");
return ret;
}
ret = getSubsliceTotal(topologyData.subSliceCount);
if (ret != 0) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Cannot query subslice total parameter!\n");
return ret;
}
}
hwInfo->gtSystemInfo.SliceCount = static_cast<uint32_t>(topologyData.sliceCount);
hwInfo->gtSystemInfo.SubSliceCount = static_cast<uint32_t>(topologyData.subSliceCount);
hwInfo->gtSystemInfo.DualSubSliceCount = static_cast<uint32_t>(topologyData.subSliceCount);
hwInfo->gtSystemInfo.EUCount = static_cast<uint32_t>(topologyData.euCount);
if (topologyData.maxSubSliceCount > 0) {
hwInfo->gtSystemInfo.MaxSubSlicesSupported = static_cast<uint32_t>(topologyData.maxSubSliceCount);
hwInfo->gtSystemInfo.MaxDualSubSlicesSupported = static_cast<uint32_t>(topologyData.maxSubSliceCount);
}
if (topologyData.numL3Banks > 0) {
hwInfo->gtSystemInfo.L3BankCount = topologyData.numL3Banks;
}
if (hwInfo->gtSystemInfo.MaxSlicesSupported == 0) {
hwInfo->gtSystemInfo.MaxSlicesSupported = hwInfo->gtSystemInfo.SliceCount;
}
if (hwInfo->gtSystemInfo.MaxEuPerSubSlice == 0 && hwInfo->gtSystemInfo.SubSliceCount != 0) {
hwInfo->gtSystemInfo.MaxEuPerSubSlice = hwInfo->gtSystemInfo.EUCount / hwInfo->gtSystemInfo.SubSliceCount;
}
status = querySystemInfo();
auto releaseHelper = rootDeviceEnvironment.getReleaseHelper();
device->setupHardwareInfo(hwInfo, setupFeatureTableAndWorkaroundTable, releaseHelper);
rootDeviceEnvironment.setRcsExposure();
if (status) {
systemInfo->checkSysInfoMismatch(hwInfo);
setupSystemInfo(hwInfo, systemInfo.get());
uint32_t bankCount = (hwInfo->gtSystemInfo.L3BankCount > 0) ? hwInfo->gtSystemInfo.L3BankCount : hwInfo->gtSystemInfo.MaxDualSubSlicesSupported;
hwInfo->gtSystemInfo.L3CacheSizeInKb = systemInfo->getL3BankSizeInKb() * bankCount;
}
if (!hwInfo->gtSystemInfo.EUCount) {
return -1;
}
setupCacheInfo(*hwInfo);
hwInfo->capabilityTable.deviceName = device->devName;
return 0;
}
void appendHwDeviceId(std::vector<std::unique_ptr<HwDeviceId>> &hwDeviceIds, int fileDescriptor, const char *pciPath, const char *devNodePath) {
if (fileDescriptor >= 0) {
if (Drm::isDrmSupported(fileDescriptor)) {
hwDeviceIds.push_back(std::make_unique<HwDeviceIdDrm>(fileDescriptor, pciPath, devNodePath));
} else {
SysCalls::close(fileDescriptor);
}
}
}
std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevices(ExecutionEnvironment &executionEnvironment) {
std::string str = "";
return Drm::discoverDevices(executionEnvironment, str);
}
std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevice(ExecutionEnvironment &executionEnvironment, std::string &osPciPath) {
return Drm::discoverDevices(executionEnvironment, osPciPath);
}
std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevices(ExecutionEnvironment &executionEnvironment, std::string &osPciPath) {
std::vector<std::unique_ptr<HwDeviceId>> hwDeviceIds;
executionEnvironment.osEnvironment = std::make_unique<OsEnvironment>();
size_t numRootDevices = 0u;
if (debugManager.flags.CreateMultipleRootDevices.get()) {
numRootDevices = debugManager.flags.CreateMultipleRootDevices.get();
}
std::vector<std::string> files = Directory::getFiles(Os::pciDevicesDirectory);
if (files.size() == 0) {
const char *pathPrefix = "/dev/dri/renderD";
const unsigned int maxDrmDevices = 64;
unsigned int startNum = 128;
for (unsigned int i = 0; i < maxDrmDevices; i++) {
std::string path = std::string(pathPrefix) + std::to_string(i + startNum);
int fileDescriptor = SysCalls::open(path.c_str(), O_RDWR | O_CLOEXEC);
if (fileDescriptor < 0) {
continue;
}
auto pciPath = NEO::getPciPath(fileDescriptor);
appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.value_or("0000:00:02.0").c_str(), path.c_str());
if (!hwDeviceIds.empty() && hwDeviceIds.size() == numRootDevices) {
break;
}
}
return hwDeviceIds;
}
do {
const char *renderDeviceSuffix = "-render";
for (std::vector<std::string>::iterator file = files.begin(); file != files.end(); ++file) {
std::string_view devicePathView(file->c_str(), file->size());
devicePathView = devicePathView.substr(strlen(Os::pciDevicesDirectory));
auto rdsPos = devicePathView.rfind(renderDeviceSuffix);
if (rdsPos == std::string::npos) {
continue;
}
if (rdsPos < devicePathView.size() - strlen(renderDeviceSuffix)) {
continue;
}
// at least 'pci-0000:00:00.0' -> 16
if (rdsPos < 16 || devicePathView[rdsPos - 13] != '-') {
continue;
}
std::string pciPath(devicePathView.substr(rdsPos - 12, 12));
if (!osPciPath.empty()) {
if (osPciPath.compare(pciPath) != 0) {
// if osPciPath is non-empty, then interest is only in discovering device having same bdf as ocPciPath. Skip all other devices.
continue;
}
}
if (debugManager.flags.FilterBdfPath.get() != "unk") {
if (devicePathView.find(debugManager.flags.FilterBdfPath.get().c_str()) == std::string::npos) {
continue;
}
}
int fileDescriptor = SysCalls::open(file->c_str(), O_RDWR | O_CLOEXEC);
appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.c_str(), file->c_str());
if (!hwDeviceIds.empty() && hwDeviceIds.size() == numRootDevices) {
break;
}
}
if (hwDeviceIds.empty()) {
return hwDeviceIds;
}
} while (hwDeviceIds.size() < numRootDevices);
return hwDeviceIds;
}
std::string Drm::getDrmVersion(int fileDescriptor) {
DrmVersion version = {};
char name[5] = {};
version.name = name;
version.nameLen = 5;
auto requestValue = getIoctlRequestValue(DrmIoctl::version, nullptr);
int ret = SysCalls::ioctl(fileDescriptor, requestValue, &version);
if (ret) {
return {};
}
name[4] = '\0';
return std::string(name);
}
template <typename DataType>
std::vector<DataType> Drm::query(uint32_t queryId, uint32_t queryItemFlags) {
Query query{};
QueryItem queryItem{};
queryItem.queryId = queryId;
queryItem.length = 0; // query length first
queryItem.flags = queryItemFlags;
query.itemsPtr = reinterpret_cast<uint64_t>(&queryItem);
query.numItems = 1;
auto ret = ioctlHelper->ioctl(DrmIoctl::query, &query);
if (ret != 0 || queryItem.length <= 0) {
return {};
}
auto data = std::vector<DataType>(Math::divideAndRoundUp(queryItem.length, sizeof(DataType)), 0);
queryItem.dataPtr = castToUint64(data.data());
ret = ioctlHelper->ioctl(DrmIoctl::query, &query);
if (ret != 0 || queryItem.length <= 0) {
return {};
}
return data;
}
void Drm::printIoctlStatistics() {
if (!debugManager.flags.PrintKmdTimes.get()) {
return;
}
printf("\n--- Ioctls statistics ---\n");
printf("%41s %15s %10s %20s %20s %20s", "Request", "Total time(ns)", "Count", "Avg time per ioctl", "Min", "Max\n");
for (const auto &ioctlData : this->ioctlStatistics) {
printf("%41s %15llu %10lu %20f %20lld %20lld\n",
getIoctlString(ioctlData.first, ioctlHelper.get()).c_str(),
ioctlData.second.totalTime,
static_cast<unsigned long>(ioctlData.second.count),
ioctlData.second.totalTime / static_cast<double>(ioctlData.second.count),
ioctlData.second.minTime,
ioctlData.second.maxTime);
}
printf("\n");
}
bool Drm::createVirtualMemoryAddressSpace(uint32_t vmCount) {
for (auto i = 0u; i < vmCount; i++) {
uint32_t id = i;
if (0 != createDrmVirtualMemory(id)) {
return false;
}
virtualMemoryIds.push_back(id);
}
return true;
}
void Drm::destroyVirtualMemoryAddressSpace() {
for (auto id : virtualMemoryIds) {
destroyDrmVirtualMemory(id);
}
virtualMemoryIds.clear();
}
uint32_t Drm::getVirtualMemoryAddressSpace(uint32_t vmId) const {
if (vmId < virtualMemoryIds.size()) {
return virtualMemoryIds[vmId];
}
return 0;
}
void Drm::setNewResourceBoundToVM(BufferObject *bo, uint32_t vmHandleId) {
if (!this->rootDeviceEnvironment.getProductHelper().isTlbFlushRequired()) {
return;
}
const auto &engines = this->rootDeviceEnvironment.executionEnvironment.memoryManager->getRegisteredEngines(bo->getRootDeviceIndex());
for (const auto &engine : engines) {
if (engine.osContext->getDeviceBitfield().test(vmHandleId)) {
auto osContextLinux = static_cast<OsContextLinux *>(engine.osContext);
osContextLinux->setNewResourceBound();
}
}
}
PhysicalDevicePciBusInfo Drm::getPciBusInfo() const {
PhysicalDevicePciBusInfo pciBusInfo(PhysicalDevicePciBusInfo::invalidValue, PhysicalDevicePciBusInfo::invalidValue, PhysicalDevicePciBusInfo::invalidValue, PhysicalDevicePciBusInfo::invalidValue);
if (adapterBDF.Data != std::numeric_limits<uint32_t>::max()) {
pciBusInfo.pciDomain = this->pciDomain;
pciBusInfo.pciBus = adapterBDF.Bus;
pciBusInfo.pciDevice = adapterBDF.Device;
pciBusInfo.pciFunction = adapterBDF.Function;
}
return pciBusInfo;
}
void Drm::cleanup() {
destroyVirtualMemoryAddressSpace();
}
Drm::~Drm() {
this->printIoctlStatistics();
}
int Drm::queryAdapterBDF() {
constexpr int pciBusInfoTokensNum = 4;
uint16_t domain = -1;
uint8_t bus = -1, device = -1, function = -1;
if (NEO::parseBdfString(hwDeviceId->getPciPath(), domain, bus, device, function) != pciBusInfoTokensNum) {
adapterBDF.Data = std::numeric_limits<uint32_t>::max();
return 1;
}
setPciDomain(domain);
adapterBDF.Bus = bus;
adapterBDF.Function = function;
adapterBDF.Device = device;
return 0;
}
void Drm::setGmmInputArgs(void *args) {
auto gmmInArgs = reinterpret_cast<GMM_INIT_IN_ARGS *>(args);
#if defined(__linux__)
gmmInArgs->FileDescriptor = adapterBDF.Data;
#endif
gmmInArgs->ClientType = GMM_CLIENT::GMM_OCL_VISTA;
}
const std::vector<int> &Drm::getSliceMappings(uint32_t deviceIndex) {
return topologyMap[deviceIndex].sliceIndices;
}
int Drm::waitHandle(uint32_t waitHandle, int64_t timeout) {
UNRECOVERABLE_IF(isVmBindAvailable());
GemWait wait{};
wait.boHandle = waitHandle;
wait.timeoutNs = timeout;
int ret = ioctlHelper->ioctl(DrmIoctl::gemWait, &wait);
if (ret != 0) {
int err = errno;
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(I915_GEM_WAIT) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
}
return ret;
}
int Drm::getTimestampFrequency(int &frequency) {
frequency = 0;
return getParamIoctl(DrmParam::paramCsTimestampFrequency, &frequency);
}
int Drm::getOaTimestampFrequency(int &frequency) {
frequency = 0;
return getParamIoctl(DrmParam::paramOATimestampFrequency, &frequency);
}
bool Drm::queryEngineInfo() {
return Drm::queryEngineInfo(false);
}
bool Drm::sysmanQueryEngineInfo() {
return Drm::queryEngineInfo(true);
}
bool Drm::isDebugAttachAvailable() {
int enableEuDebug = getEuDebugSysFsEnable();
return (enableEuDebug == 1) && ioctlHelper->isDebugAttachAvailable();
}
int Drm::getEuDebugSysFsEnable() {
return ioctlHelper->getEuDebugSysFsEnable();
}
int getMaxGpuFrequencyOfDevice(Drm &drm, std::string &sysFsPciPath, int &maxGpuFrequency) {
maxGpuFrequency = 0;
std::string clockSysFsPath = sysFsPciPath + drm.getIoctlHelper()->getFileForMaxGpuFrequency();
std::ifstream ifs(clockSysFsPath.c_str(), std::ifstream::in);
if (ifs.fail()) {
return -1;
}
ifs >> maxGpuFrequency;
ifs.close();
return 0;
}
int getMaxGpuFrequencyOfSubDevice(Drm &drm, std::string &sysFsPciPath, int subDeviceId, int &maxGpuFrequency) {
maxGpuFrequency = 0;
std::string clockSysFsPath = sysFsPciPath + drm.getIoctlHelper()->getFileForMaxGpuFrequencyOfSubDevice(subDeviceId);
std::ifstream ifs(clockSysFsPath.c_str(), std::ifstream::in);
if (ifs.fail()) {
return -1;
}
ifs >> maxGpuFrequency;
ifs.close();
return 0;
}
int Drm::getMaxGpuFrequency(HardwareInfo &hwInfo, int &maxGpuFrequency) {
int ret = 0;
std::string sysFsPciPath = getSysFsPciPath();
auto tileCount = hwInfo.gtSystemInfo.MultiTileArchInfo.TileCount;
if (hwInfo.gtSystemInfo.MultiTileArchInfo.IsValid && tileCount > 0) {
for (auto tileId = 0; tileId < tileCount; tileId++) {
int maxGpuFreqOfSubDevice = 0;
ret |= getMaxGpuFrequencyOfSubDevice(*this, sysFsPciPath, tileId, maxGpuFreqOfSubDevice);
maxGpuFrequency = std::max(maxGpuFrequency, maxGpuFreqOfSubDevice);
}
if (ret == 0) {
return 0;
}
}
return getMaxGpuFrequencyOfDevice(*this, sysFsPciPath, maxGpuFrequency);
}
bool Drm::getDeviceMemoryMaxClockRateInMhz(uint32_t tileId, uint32_t &clkRate) {
const std::string relativefilePath = ioctlHelper->getFileForMaxMemoryFrequencyOfSubDevice(tileId);
std::string readString(64, '\0');
errno = 0;
if (readSysFsAsString(relativefilePath, readString) == false) {
return false;
}
char *endPtr = nullptr;
uint32_t retClkRate = static_cast<uint32_t>(std::strtoul(readString.data(), &endPtr, 10));
if ((endPtr == readString.data()) || (errno != 0)) {
return false;
}
clkRate = retClkRate;
return true;
}
bool Drm::getDeviceMemoryPhysicalSizeInBytes(uint32_t tileId, uint64_t &physicalSize) {
const std::string relativefilePath = "/gt/gt" + std::to_string(tileId) + "/addr_range";
std::string readString(64, '\0');
errno = 0;
if (readSysFsAsString(relativefilePath, readString) == false) {
return false;
}
char *endPtr = nullptr;
uint64_t retSize = static_cast<uint64_t>(std::strtoull(readString.data(), &endPtr, 16));
if ((endPtr == readString.data()) || (errno != 0)) {
return false;
}
physicalSize = retSize;
return true;
}
bool Drm::useVMBindImmediate() const {
bool useBindImmediate = isDirectSubmissionActive() || hasPageFaultSupport() || ioctlHelper->isImmediateVmBindRequired();
if (debugManager.flags.EnableImmediateVmBindExt.get() != -1) {
useBindImmediate = debugManager.flags.EnableImmediateVmBindExt.get();
}
return useBindImmediate;
}
void Drm::setupSystemInfo(HardwareInfo *hwInfo, SystemInfo *sysInfo) {
GT_SYSTEM_INFO *gtSysInfo = &hwInfo->gtSystemInfo;
gtSysInfo->MaxEuPerSubSlice = sysInfo->getMaxEuPerDualSubSlice();
if (!gtSysInfo->EUCount || gtSysInfo->EUCount > gtSysInfo->SubSliceCount * gtSysInfo->MaxEuPerSubSlice) {
gtSysInfo->EUCount = gtSysInfo->SubSliceCount * gtSysInfo->MaxEuPerSubSlice;
}
gtSysInfo->ThreadCount = gtSysInfo->EUCount * sysInfo->getNumThreadsPerEu();
gtSysInfo->MaxSlicesSupported = sysInfo->getMaxSlicesSupported();
gtSysInfo->MaxSubSlicesSupported = sysInfo->getMaxDualSubSlicesSupported();
gtSysInfo->MaxDualSubSlicesSupported = sysInfo->getMaxDualSubSlicesSupported();
UNRECOVERABLE_IF(gtSysInfo->MaxSlicesSupported == 0);
auto maxDssPerSlice = gtSysInfo->MaxDualSubSlicesSupported / gtSysInfo->MaxSlicesSupported;
gtSysInfo->SliceCount = static_cast<uint32_t>(Math::divideAndRoundUp(gtSysInfo->DualSubSliceCount, maxDssPerSlice));
gtSysInfo->MemoryType = sysInfo->getMemoryType();
gtSysInfo->CsrSizeInMb = sysInfo->getCsrSizeInMb();
gtSysInfo->SLMSizeInKb = sysInfo->getSlmSizePerDss();
if (!hwInfo->capabilityTable.slmSize) {
hwInfo->capabilityTable.slmSize = gtSysInfo->SLMSizeInKb;
}
}
void Drm::setupCacheInfo(const HardwareInfo &hwInfo) {
auto &gfxCoreHelper = rootDeviceEnvironment.getHelper<GfxCoreHelper>();
if (debugManager.flags.ClosEnabled.get() == 0 || gfxCoreHelper.getNumCacheRegions() == 0) {
this->cacheInfo.reset(new CacheInfo(*this, 0, 0, 0));
return;
}
const GT_SYSTEM_INFO *gtSysInfo = &hwInfo.gtSystemInfo;
constexpr uint16_t maxNumWays = 32;
constexpr uint16_t globalReservationLimit = 16;
constexpr uint16_t clientReservationLimit = 8;
constexpr uint16_t maxReservationNumWays = std::min(globalReservationLimit, clientReservationLimit);
const size_t totalCacheSize = gtSysInfo->L3CacheSizeInKb * MemoryConstants::kiloByte;
const size_t maxReservationCacheSize = (totalCacheSize * maxReservationNumWays) / maxNumWays;
const uint32_t maxReservationNumCacheRegions = gfxCoreHelper.getNumCacheRegions() - 1;
this->cacheInfo.reset(new CacheInfo(*this, maxReservationCacheSize, maxReservationNumCacheRegions, maxReservationNumWays));
}
void Drm::getPrelimVersion(std::string &prelimVersion) {
std::string sysFsPciPath = getSysFsPciPath();
std::string prelimVersionPath = sysFsPciPath + "/prelim_uapi_version";
std::ifstream ifs(prelimVersionPath.c_str(), std::ifstream::in);
if (ifs.fail()) {
prelimVersion = "";
} else {
ifs >> prelimVersion;
}
ifs.close();
}
int Drm::waitUserFence(uint32_t ctxId, uint64_t address, uint64_t value, ValueWidth dataWidth, int64_t timeout, uint16_t flags, bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) {
return ioctlHelper->waitUserFence(ctxId, address, value, static_cast<uint32_t>(dataWidth), timeout, flags, userInterrupt, externalInterruptId, allocForInterruptWait);
}
bool Drm::querySystemInfo() {
auto request = ioctlHelper->getDrmParamValue(DrmParam::queryHwconfigTable);
auto deviceBlobQuery = this->query<uint32_t>(request, 0);
if (deviceBlobQuery.empty()) {
PRINT_DEBUG_STRING(debugManager.flags.PrintDebugMessages.get(), stdout, "%s", "INFO: System Info query failed!\n");
return false;
}
this->systemInfo.reset(new SystemInfo(deviceBlobQuery));
return true;
}
std::vector<uint64_t> Drm::getMemoryRegions() {
auto request = ioctlHelper->getDrmParamValue(DrmParam::queryMemoryRegions);
return this->query<uint64_t>(request, 0);
}
bool Drm::queryMemoryInfo() {
this->memoryInfo = ioctlHelper->createMemoryInfo();
return this->memoryInfo != nullptr;
}
bool Drm::queryEngineInfo(bool isSysmanEnabled) {
this->engineInfo = ioctlHelper->createEngineInfo(isSysmanEnabled);
if (this->engineInfo && (this->engineInfo->hasEngines() == false)) {
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "FATAL: Engine info size is equal to 0.\n");
}
return this->engineInfo != nullptr;
}
bool Drm::completionFenceSupport() {
std::call_once(checkCompletionFenceOnce, [this]() {
const bool vmBindAvailable = isVmBindAvailable();
bool support = ioctlHelper->completionFenceExtensionSupported(vmBindAvailable);
int32_t overrideCompletionFence = debugManager.flags.EnableDrmCompletionFence.get();
if (overrideCompletionFence != -1) {
support = !!overrideCompletionFence;
}
completionFenceSupported = support;
if (debugManager.flags.PrintCompletionFenceUsage.get()) {
std::cout << "Completion fence supported: " << completionFenceSupported << std::endl;
}
});
return completionFenceSupported;
}
void Drm::setupIoctlHelper(const PRODUCT_FAMILY productFamily) {
if (!this->ioctlHelper) {
std::string prelimVersion = "";
getPrelimVersion(prelimVersion);
this->ioctlHelper = IoctlHelper::getI915Helper(productFamily, prelimVersion, *this);
this->ioctlHelper->initialize();
}
}
bool Drm::queryTopology(const HardwareInfo &hwInfo, DrmQueryTopologyData &topologyData) {
auto result = this->ioctlHelper->getTopologyDataAndMap(hwInfo, topologyData, topologyMap);
return result;
}
void Drm::queryPageFaultSupport() {
const auto &productHelper = this->getRootDeviceEnvironment().getHelper<ProductHelper>();
if (!productHelper.isPageFaultSupported()) {
return;
}
pageFaultSupported = this->ioctlHelper->isPageFaultSupported();
}
bool Drm::hasPageFaultSupport() const {
if (debugManager.flags.EnableRecoverablePageFaults.get() != -1) {
return !!debugManager.flags.EnableRecoverablePageFaults.get();
}
return pageFaultSupported;
}
bool Drm::hasKmdMigrationSupport() const {
const auto &productHelper = this->getRootDeviceEnvironment().getHelper<ProductHelper>();
auto kmdMigrationSupported = hasPageFaultSupport() && productHelper.isKmdMigrationSupported();
if (debugManager.flags.UseKmdMigration.get() != -1) {
return !!debugManager.flags.UseKmdMigration.get();
}
return kmdMigrationSupported;
}
void Drm::configureScratchPagePolicy() {
if (debugManager.flags.DisableScratchPages.get() != -1) {
disableScratch = !!debugManager.flags.DisableScratchPages.get();
return;
}
const auto &productHelper = this->getRootDeviceEnvironment().getHelper<ProductHelper>();
disableScratch = (productHelper.isDisableScratchPagesSupported() &&
!rootDeviceEnvironment.executionEnvironment.isDebuggingEnabled());
}
void Drm::configureGpuFaultCheckThreshold() {
if (debugManager.flags.GpuFaultCheckThreshold.get() != -1) {
gpuFaultCheckThreshold = debugManager.flags.GpuFaultCheckThreshold.get();
}
}
unsigned int Drm::bindDrmContext(uint32_t drmContextId, uint32_t deviceIndex, aub_stream::EngineType engineType, bool engineInstancedDevice) {
auto engineInfo = this->engineInfo.get();
auto retVal = static_cast<unsigned int>(ioctlHelper->getDrmParamValue(DrmEngineMapper::engineNodeMap(engineType)));
if (!engineInfo) {
return retVal;
}
auto engine = engineInfo->getEngineInstance(deviceIndex, engineType);
if (!engine) {
return retVal;
}
bool useVirtualEnginesForCcs = !engineInstancedDevice;
if (debugManager.flags.UseDrmVirtualEnginesForCcs.get() != -1) {
useVirtualEnginesForCcs = !!debugManager.flags.UseDrmVirtualEnginesForCcs.get();
}
auto numberOfCCS = rootDeviceEnvironment.getHardwareInfo()->gtSystemInfo.CCSInfo.NumberOfCCSEnabled;
constexpr uint32_t maxEngines = 9u;
bool useVirtualEnginesForBcs = EngineHelpers::isBcsVirtualEngineEnabled(engineType);
auto numberOfBCS = rootDeviceEnvironment.getHardwareInfo()->featureTable.ftrBcsInfo.count();
if (debugManager.flags.LimitEngineCountForVirtualBcs.get() != -1) {
numberOfBCS = debugManager.flags.LimitEngineCountForVirtualBcs.get();
}
if (debugManager.flags.LimitEngineCountForVirtualCcs.get() != -1) {
numberOfCCS = debugManager.flags.LimitEngineCountForVirtualCcs.get();
}
uint32_t numEnginesInContext = 1;
ContextParamEngines<> contextEngines{};
ContextEnginesLoadBalance<maxEngines> balancer{};
ioctlHelper->insertEngineToContextParams(contextEngines, 0u, engine, deviceIndex, false);
bool setupVirtualEngines = false;
unsigned int engineCount = static_cast<unsigned int>(numberOfCCS);
if (useVirtualEnginesForCcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCompute) && numberOfCCS > 1u) {
numEnginesInContext = numberOfCCS + 1;
balancer.numSiblings = numberOfCCS;
setupVirtualEngines = true;
}
bool includeMainCopyEngineInGroup = false;
if (useVirtualEnginesForBcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCopy) && numberOfBCS > 1u) {
numEnginesInContext = static_cast<uint32_t>(numberOfBCS) + 1;
balancer.numSiblings = numberOfBCS;
setupVirtualEngines = true;
engineCount = static_cast<unsigned int>(rootDeviceEnvironment.getHardwareInfo()->featureTable.ftrBcsInfo.size());
if (EngineHelpers::getBcsIndex(engineType) == 0u) {
includeMainCopyEngineInGroup = true;
} else {
engineCount--;
balancer.numSiblings = numberOfBCS - 1;
numEnginesInContext = static_cast<uint32_t>(numberOfBCS);
}
}
if (setupVirtualEngines) {
balancer.base.name = ioctlHelper->getDrmParamValue(DrmParam::contextEnginesExtLoadBalance);
contextEngines.extensions = castToUint64(&balancer);
ioctlHelper->insertEngineToContextParams(contextEngines, 0u, nullptr, deviceIndex, true);
for (auto engineIndex = 0u; engineIndex < engineCount; engineIndex++) {
if (useVirtualEnginesForBcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCopy)) {
auto mappedBcsEngineType = static_cast<aub_stream::EngineType>(EngineHelpers::mapBcsIndexToEngineType(engineIndex, includeMainCopyEngineInGroup));
bool isBcsEnabled = rootDeviceEnvironment.getHardwareInfo()->featureTable.ftrBcsInfo.test(EngineHelpers::getBcsIndex(mappedBcsEngineType));
if (!isBcsEnabled) {
continue;
}
engine = engineInfo->getEngineInstance(deviceIndex, mappedBcsEngineType);
}
UNRECOVERABLE_IF(!engine);
if (useVirtualEnginesForCcs && engine->engineClass == ioctlHelper->getDrmParamValue(DrmParam::engineClassCompute)) {
engine = engineInfo->getEngineInstance(deviceIndex, static_cast<aub_stream::EngineType>(EngineHelpers::mapCcsIndexToEngineType(engineIndex)));
}
UNRECOVERABLE_IF(!engine);
balancer.engines[engineIndex] = {engine->engineClass, engine->engineInstance};
ioctlHelper->insertEngineToContextParams(contextEngines, engineIndex, engine, deviceIndex, true);
}
}
GemContextParam param{};
param.contextId = drmContextId;
param.size = static_cast<uint32_t>(ptrDiff(contextEngines.enginesData, &contextEngines) + sizeof(EngineClassInstance) * numEnginesInContext);
param.param = ioctlHelper->getDrmParamValue(DrmParam::contextParamEngines);
param.value = castToUint64(&contextEngines);
auto ioctlValue = ioctlHelper->ioctl(DrmIoctl::gemContextSetparam, &param);
UNRECOVERABLE_IF(ioctlValue != 0);
retVal = static_cast<unsigned int>(ioctlHelper->getDrmParamValue(DrmParam::execDefault));
return retVal;
}
void Drm::waitForBind(uint32_t vmHandleId) {
if (pagingFence[vmHandleId] >= fenceVal[vmHandleId]) {
return;
}
auto lock = this->lockBindFenceMutex();
auto fenceAddress = castToUint64(&this->pagingFence[vmHandleId]);
auto fenceValue = this->fenceVal[vmHandleId];
lock.unlock();
waitUserFence(0u, fenceAddress, fenceValue, ValueWidth::u64, -1, ioctlHelper->getWaitUserFenceSoftFlag(), false, NEO::InterruptId::notUsed, nullptr);
}
bool Drm::isSetPairAvailable() {
if (debugManager.flags.EnableSetPair.get() == 1) {
std::call_once(checkSetPairOnce, [this]() {
int ret = ioctlHelper->isSetPairAvailable();
setPairAvailable = ret;
});
}
return setPairAvailable;
}
bool Drm::isChunkingAvailable() {
if (debugManager.flags.EnableBOChunking.get() != 0) {
std::call_once(checkChunkingOnce, [this]() {
int ret = ioctlHelper->isChunkingAvailable();
if (ret) {
if (debugManager.flags.EnableBOChunking.get() == -1) {
chunkingMode = chunkingModeDevice;
} else {
chunkingMode = debugManager.flags.EnableBOChunking.get();
if (!(hasKmdMigrationSupport())) {
chunkingMode &= (~(chunkingModeShared));
}
}
}
if (chunkingMode > 0) {
chunkingAvailable = true;
}
if (debugManager.flags.MinimalAllocationSizeForChunking.get() != -1) {
minimalChunkingSize = debugManager.flags.MinimalAllocationSizeForChunking.get();
}
printDebugString(debugManager.flags.PrintBOChunkingLogs.get(), stdout,
"Chunking available: %d; enabled for: shared allocations %d, device allocations %d; minimalChunkingSize: %zd\n",
chunkingAvailable,
(chunkingMode & chunkingModeShared),
(chunkingMode & chunkingModeDevice),
minimalChunkingSize);
});
}
return chunkingAvailable;
}
bool Drm::isVmBindAvailable() {
std::call_once(checkBindOnce, [this]() {
bindAvailable = ioctlHelper->isVmBindAvailable();
Drm::overrideBindSupport(bindAvailable);
queryAndSetVmBindPatIndexProgrammingSupport();
});
return bindAvailable;
}
uint64_t Drm::getPatIndex(Gmm *gmm, AllocationType allocationType, CacheRegion cacheRegion, CachePolicy cachePolicy, bool closEnabled, bool isSystemMemory) const {
if ((debugManager.flags.OverridePatIndexForSystemMemory.get() != -1) && isSystemMemory) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForSystemMemory.get());
}
if ((debugManager.flags.OverridePatIndexForDeviceMemory.get() != -1) && !isSystemMemory) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForDeviceMemory.get());
}
if (debugManager.flags.OverridePatIndex.get() != -1) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndex.get());
}
auto &productHelper = rootDeviceEnvironment.getProductHelper();
GMM_RESOURCE_USAGE_TYPE usageType = CacheSettingsHelper::getGmmUsageType(allocationType, false, productHelper);
auto isUncachedType = CacheSettingsHelper::isUncachedType(usageType);
if (isUncachedType && debugManager.flags.OverridePatIndexForUncachedTypes.get() != -1) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForUncachedTypes.get());
}
if (!isUncachedType && debugManager.flags.OverridePatIndexForCachedTypes.get() != -1) {
return static_cast<uint64_t>(debugManager.flags.OverridePatIndexForCachedTypes.get());
}
if (!this->vmBindPatIndexProgrammingSupported) {
return CommonConstants::unsupportedPatIndex;
}
auto &gfxCoreHelper = rootDeviceEnvironment.getHelper<GfxCoreHelper>();
GMM_RESOURCE_INFO *resourceInfo = nullptr;
bool cachable = !CacheSettingsHelper::isUncachedType(usageType);
bool compressed = false;
if (gmm) {
resourceInfo = gmm->gmmResourceInfo->peekGmmResourceInfo();
usageType = gmm->resourceParams.Usage;
compressed = gmm->isCompressionEnabled();
cachable = gmm->gmmResourceInfo->getResourceFlags()->Info.Cacheable;
}
uint64_t patIndex = rootDeviceEnvironment.getGmmClientContext()->cachePolicyGetPATIndex(resourceInfo, usageType, compressed, cachable);
patIndex = productHelper.overridePatIndex(isUncachedType, patIndex, allocationType);
UNRECOVERABLE_IF(patIndex == static_cast<uint64_t>(GMM_PAT_ERROR));
if (debugManager.flags.ClosEnabled.get() != -1) {
closEnabled = !!debugManager.flags.ClosEnabled.get();
}
if (closEnabled) {
patIndex = gfxCoreHelper.getPatIndex(cacheRegion, cachePolicy);
}
return patIndex;
}
void programUserFence(Drm *drm, OsContext *osContext, BufferObject *bo, VmBindExtUserFenceT &vmBindExtUserFence, uint32_t vmHandleId, uint64_t nextExtension) {
auto ioctlHelper = drm->getIoctlHelper();
uint64_t address = 0;
uint64_t value = 0;
if (drm->isPerContextVMRequired()) {
auto osContextLinux = static_cast<OsContextLinux *>(osContext);
address = castToUint64(osContextLinux->getFenceAddr(vmHandleId));
value = osContextLinux->getNextFenceVal(vmHandleId);
} else {
address = castToUint64(drm->getFenceAddr(vmHandleId));
value = drm->getNextFenceVal(vmHandleId);
}
ioctlHelper->fillVmBindExtUserFence(vmBindExtUserFence, address, value, nextExtension);
}
int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleId, BufferObject *bo, bool bind) {
auto vmId = drm->getVirtualMemoryAddressSpace(vmHandleId);
auto ioctlHelper = drm->getIoctlHelper();
uint64_t flags = 0u;
if (drm->isPerContextVMRequired()) {
auto osContextLinux = static_cast<const OsContextLinux *>(osContext);
UNRECOVERABLE_IF(osContextLinux->getDrmVmIds().size() <= vmHandleId);
vmId = osContextLinux->getDrmVmIds()[vmHandleId];
}
std::unique_ptr<uint8_t[]> extensions;
if (bind) {
bool allowUUIDsForDebug = !osContext->isInternalEngine() && !EngineHelpers::isBcs(osContext->getEngineType());
if (bo->getBindExtHandles().size() > 0 && allowUUIDsForDebug) {
extensions = ioctlHelper->prepareVmBindExt(bo->getBindExtHandles());
}
bool bindCapture = bo->isMarkedForCapture();
bool bindImmediate = bo->isImmediateBindingRequired();
bool bindMakeResident = false;
bool readOnlyResource = bo->isReadOnlyGpuResource();
if (drm->useVMBindImmediate()) {
bindMakeResident = bo->isExplicitResidencyRequired();
bindImmediate = true;
}
bool bindLock = bo->isExplicitLockedMemoryRequired();
flags |= ioctlHelper->getFlagsForVmBind(bindCapture, bindImmediate, bindMakeResident, bindLock, readOnlyResource);
}
auto &bindAddresses = bo->getColourAddresses();
auto bindIterations = bindAddresses.size();
if (bindIterations == 0) {
bindIterations = 1;
}
int ret = 0;
for (size_t i = 0; i < bindIterations; i++) {
VmBindParams vmBind{};
vmBind.vmId = static_cast<uint32_t>(vmId);
vmBind.flags = flags;
vmBind.handle = bo->peekHandle();
vmBind.length = bo->peekSize();
vmBind.offset = 0;
vmBind.start = bo->peekAddress();
vmBind.userptr = bo->getUserptr();
if (bo->getColourWithBind()) {
vmBind.length = bo->getColourChunk();
vmBind.offset = bo->getColourChunk() * i;
vmBind.start = bindAddresses[i];
}
VmBindExtSetPatT vmBindExtSetPat{};
if (drm->isVmBindPatIndexProgrammingSupported()) {
UNRECOVERABLE_IF(bo->peekPatIndex() == CommonConstants::unsupportedPatIndex);
if (ioctlHelper->isVmBindPatIndexExtSupported()) {
ioctlHelper->fillVmBindExtSetPat(vmBindExtSetPat, bo->peekPatIndex(), castToUint64(extensions.get()));
vmBind.extensions = castToUint64(vmBindExtSetPat);
} else {
vmBind.extensions = castToUint64(extensions.get());
}
vmBind.patIndex = bo->peekPatIndex();
} else {
vmBind.extensions = castToUint64(extensions.get());
}
std::unique_lock<std::mutex> lock;
VmBindExtUserFenceT vmBindExtUserFence{};
bool incrementFenceValue = false;
if (ioctlHelper->isWaitBeforeBindRequired(bind)) {
if (drm->useVMBindImmediate()) {
lock = drm->lockBindFenceMutex();
auto nextExtension = vmBind.extensions;
incrementFenceValue = true;
programUserFence(drm, osContext, bo, vmBindExtUserFence, vmHandleId, nextExtension);
ioctlHelper->setVmBindUserFence(vmBind, vmBindExtUserFence);
}
}
if (bind) {
ret = ioctlHelper->vmBind(vmBind);
if (ret) {
break;
}
drm->setNewResourceBoundToVM(bo, vmHandleId);
} else {
vmBind.handle = 0u;
ret = ioctlHelper->vmUnbind(vmBind);
if (ret) {
break;
}
}
bool waitOnUserFenceAfterBindAndUnbind = false;
if (debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get() != -1) {
waitOnUserFenceAfterBindAndUnbind = !!debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get();
}
if (ioctlHelper->isWaitBeforeBindRequired(bind) && waitOnUserFenceAfterBindAndUnbind && drm->useVMBindImmediate()) {
auto osContextLinux = static_cast<OsContextLinux *>(osContext);
osContextLinux->waitForPagingFence();
}
if (incrementFenceValue) {
if (drm->isPerContextVMRequired()) {
auto osContextLinux = static_cast<OsContextLinux *>(osContext);
osContextLinux->incFenceVal(vmHandleId);
} else {
drm->incFenceVal(vmHandleId);
}
}
}
return ret;
}
int Drm::bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) {
auto ret = changeBufferObjectBinding(this, osContext, vmHandleId, bo, true);
if (ret != 0) {
static_cast<DrmMemoryOperationsHandlerBind *>(this->rootDeviceEnvironment.memoryOperationsInterface.get())->evictUnusedAllocations(false, false);
ret = changeBufferObjectBinding(this, osContext, vmHandleId, bo, true);
}
return ret;
}
int Drm::unbindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) {
return changeBufferObjectBinding(this, osContext, vmHandleId, bo, false);
}
int Drm::createDrmVirtualMemory(uint32_t &drmVmId) {
GemVmControl ctl{};
std::optional<MemoryClassInstance> regionInstanceClass;
uint32_t memoryBank = 1 << drmVmId;
auto hwInfo = this->getRootDeviceEnvironment().getHardwareInfo();
auto memInfo = this->getMemoryInfo();
if (debugManager.flags.UseTileMemoryBankInVirtualMemoryCreation.get() != 0) {
if (memInfo && rootDeviceEnvironment.getHelper<GfxCoreHelper>().getEnableLocalMemory(*hwInfo)) {
regionInstanceClass = memInfo->getMemoryRegionClassAndInstance(memoryBank, *this->rootDeviceEnvironment.getHardwareInfo());
}
}
auto vmControlExtRegion = ioctlHelper->createVmControlExtRegion(regionInstanceClass);
if (vmControlExtRegion) {
ctl.extensions = castToUint64(vmControlExtRegion.get());
}
bool useVmBind = isVmBindAvailable();
bool enablePageFault = hasPageFaultSupport() && useVmBind;
ctl.flags = ioctlHelper->getFlagsForVmCreate(checkToDisableScratchPage(), enablePageFault, useVmBind);
auto ret = ioctlHelper->ioctl(DrmIoctl::gemVmCreate, &ctl);
if (ret == 0) {
drmVmId = ctl.vmId;
if (ctl.vmId == 0) {
// 0 is reserved for invalid/unassigned ppgtt
return -1;
}
} else {
printDebugString(debugManager.flags.PrintDebugMessages.get(), stderr,
"INFO: Cannot create Virtual Memory at memory bank 0x%x info present %d return code %d\n",
memoryBank, memoryInfo != nullptr, ret);
}
return ret;
}
PhysicalDevicePciSpeedInfo Drm::getPciSpeedInfo() const {
PhysicalDevicePciSpeedInfo pciSpeedInfo = {};
std::string pathPrefix{};
bool isIntegratedDevice = rootDeviceEnvironment.getHardwareInfo()->capabilityTable.isIntegratedDevice;
// If integrated device, read properties from the specific device path.
// If discrete device, read properties from the root path of the pci device.
if (isIntegratedDevice) {
auto devicePath = NEO::getPciLinkPath(getFileDescriptor());
if (!devicePath.has_value()) {
return pciSpeedInfo;
}
pathPrefix = "/sys/class/drm/" + devicePath.value() + "/device/";
} else {
auto rootPath = NEO::getPciRootPath(getFileDescriptor());
if (!rootPath.has_value()) {
return pciSpeedInfo;
}
pathPrefix += "/sys/devices" + rootPath.value();
}
std::array<char, 32> readString = {'\0'};
errno = 0;
auto readFile = [](const std::string fileName, const std::string_view pathPrefix, std::array<char, 32> &readString) {
std::ostringstream linkWidthStream{};
linkWidthStream << pathPrefix << fileName;
int fd = NEO::SysCalls::open(linkWidthStream.str().c_str(), O_RDONLY);
if (fd < 0) {
return false;
}
ssize_t bytesRead = NEO::SysCalls::pread(fd, readString.data(), readString.size() - 1, 0);
NEO::SysCalls::close(fd);
if (bytesRead <= 0) {
return false;
}
std::replace(readString.begin(), readString.end(), '\n', '\0');
return true;
};
// read max link width
if (readFile("/max_link_width", pathPrefix, readString) != true) {
return pciSpeedInfo;
}
char *endPtr = nullptr;
uint32_t linkWidth = static_cast<uint32_t>(std::strtoul(readString.data(), &endPtr, 10));
if ((endPtr == readString.data()) || (errno != 0)) {
return pciSpeedInfo;
}
pciSpeedInfo.width = linkWidth;
// read max link speed
if (readFile("/max_link_speed", pathPrefix, readString) != true) {
return pciSpeedInfo;
}
endPtr = nullptr;
const auto maxSpeed = strtod(readString.data(), &endPtr);
if ((endPtr == readString.data()) || (errno != 0)) {
return pciSpeedInfo;
}
double gen3EncodingLossFactor = 128.0 / 130.0;
std::map<double, std::pair<int32_t, double>> maxSpeedToGenAndEncodingLossMapping{
//{max link speed, {pci generation, encoding loss factor}}
{2.5, {1, 0.2}},
{5.0, {2, 0.2}},
{8.0, {3, gen3EncodingLossFactor}},
{16.0, {4, gen3EncodingLossFactor}},
{32.0, {5, gen3EncodingLossFactor}}};
if (maxSpeedToGenAndEncodingLossMapping.find(maxSpeed) == maxSpeedToGenAndEncodingLossMapping.end()) {
return pciSpeedInfo;
}
pciSpeedInfo.genVersion = maxSpeedToGenAndEncodingLossMapping[maxSpeed].first;
constexpr double gigaBitsPerSecondToBytesPerSecondMultiplier = 125000000;
const auto maxSpeedWithEncodingLoss = maxSpeed * gigaBitsPerSecondToBytesPerSecondMultiplier * maxSpeedToGenAndEncodingLossMapping[maxSpeed].second;
pciSpeedInfo.maxBandwidth = static_cast<int64_t>(maxSpeedWithEncodingLoss * pciSpeedInfo.width);
return pciSpeedInfo;
}
int Drm::waitOnUserFences(OsContextLinux &osContext, uint64_t address, uint64_t value, uint32_t numActiveTiles, int64_t timeout, uint32_t postSyncOffset, bool userInterrupt,
uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) {
int ret = waitOnUserFencesImpl(static_cast<const OsContextLinux &>(osContext), address, value, numActiveTiles,
timeout, postSyncOffset, userInterrupt, externalInterruptId, allocForInterruptWait);
if (ret != 0 && getErrno() == EIO && checkGpuPageFaultRequired()) {
checkResetStatus(osContext);
}
return ret;
}
int Drm::waitOnUserFencesImpl(const OsContextLinux &osContext, uint64_t address, uint64_t value, uint32_t numActiveTiles, int64_t timeout, uint32_t postSyncOffset, bool userInterrupt,
uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) {
auto &drmContextIds = osContext.getDrmContextIds();
UNRECOVERABLE_IF(numActiveTiles > drmContextIds.size());
auto completionFenceCpuAddress = address;
const auto selectedTimeout = osContext.isHangDetected() ? 1 : timeout;
for (auto drmIterator = 0u; drmIterator < numActiveTiles; drmIterator++) {
if (*reinterpret_cast<uint32_t *>(completionFenceCpuAddress) < value) {
static constexpr uint16_t flags = 0;
int retVal = waitUserFence(drmContextIds[drmIterator], completionFenceCpuAddress, value, Drm::ValueWidth::u64, selectedTimeout, flags, userInterrupt, externalInterruptId, allocForInterruptWait);
if (debugManager.flags.PrintCompletionFenceUsage.get()) {
std::cout << "Completion fence waited."
<< " Status: " << retVal
<< ", CPU address: " << std::hex << completionFenceCpuAddress << std::dec
<< ", current value: " << *reinterpret_cast<uint32_t *>(completionFenceCpuAddress)
<< ", wait value: " << value << std::endl;
}
if (retVal != 0) {
return retVal;
}
} else if (debugManager.flags.PrintCompletionFenceUsage.get()) {
std::cout << "Completion fence already completed."
<< " CPU address: " << std::hex << completionFenceCpuAddress << std::dec
<< ", current value: " << *reinterpret_cast<uint32_t *>(completionFenceCpuAddress)
<< ", wait value: " << value << std::endl;
}
if (externalInterruptId != NEO::InterruptId::notUsed) {
break;
}
completionFenceCpuAddress = ptrOffset(completionFenceCpuAddress, postSyncOffset);
}
return 0;
}
const HardwareInfo *Drm::getHardwareInfo() const { return rootDeviceEnvironment.getHardwareInfo(); }
uint64_t Drm::alignUpGttSize(uint64_t inputGttSize) {
constexpr uint64_t gttSize47bit = (1ull << 47);
constexpr uint64_t gttSize48bit = (1ull << 48);
if (inputGttSize > gttSize47bit && inputGttSize < gttSize48bit) {
return gttSize48bit;
}
return inputGttSize;
}
template std::vector<uint16_t> Drm::query<uint16_t>(uint32_t queryId, uint32_t queryItemFlags);
template std::vector<uint32_t> Drm::query<uint32_t>(uint32_t queryId, uint32_t queryItemFlags);
template std::vector<uint64_t> Drm::query<uint64_t>(uint32_t queryId, uint32_t queryItemFlags);
} // namespace NEO
Reference in New Issue View Git Blame Copy Permalink