intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-26 15:03:02 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
c294ef48ce8bb911d4b4fdc403668ea185465396
compute-runtime/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp
Filip Hazubski 6489379508 feature: Add pat index programming to gem create ext call 
When upstream ioctl helper is created it will try to create small
allocation, adding I915_GEM_CREATE_EXT_SET_PAT extension. If it
succeeds, for all resources with valid pat index value it will then
explicitly program pat index value with gem create ext call.

PrintBOCreateDestroyResult value can be used to:
- print whether the set pat extension is supported by the kernel, when
  ioctl helper is created
- print whether set pat extension was added for a given gem create ext
  call and what pat index value was programmed

Resolves: NEO-7896

Signed-off-by: Filip Hazubski <filip.hazubski@intel.com>
2023-06-29 08:51:00 +02:00

1451 lines
52 KiB
C++
Raw Blame History

/*
* Copyright (C) 2023 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/os_interface/linux/xe/ioctl_helper_xe.h"
#include "shared/source/command_stream/csr_definitions.h"
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/helpers/basic_math.h"
#include "shared/source/helpers/bit_helpers.h"
#include "shared/source/helpers/common_types.h"
#include "shared/source/helpers/constants.h"
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/helpers/register_offsets.h"
#include "shared/source/helpers/string.h"
#include "shared/source/os_interface/linux/drm_neo.h"
#include "shared/source/os_interface/linux/engine_info.h"
#include "shared/source/os_interface/linux/memory_info.h"
#include "shared/source/os_interface/linux/os_context_linux.h"
#include "drm/i915_drm_prelim.h"
#include "drm/xe_drm.h"
#include <algorithm>
#include <iostream>
#define XE_FIND_INVALID_INSTANCE 16
#define STRINGIFY_ME(X) return #X
#define RETURN_ME(X) return X
#define XE_USERPTR_FAKE_FLAG 0x800000
#define XE_USERPTR_FAKE_MASK 0x7FFFFF
#define USER_FENCE_VALUE 0xc0ffee0000000000ull
namespace NEO {
static_assert(DRM_XE_ENGINE_CLASS_RENDER == I915_ENGINE_CLASS_RENDER);
static_assert(DRM_XE_ENGINE_CLASS_COPY == I915_ENGINE_CLASS_COPY);
static_assert(DRM_XE_ENGINE_CLASS_VIDEO_DECODE == I915_ENGINE_CLASS_VIDEO);
static_assert(DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE == I915_ENGINE_CLASS_VIDEO_ENHANCE);
static_assert(DRM_XE_ENGINE_CLASS_COMPUTE == I915_ENGINE_CLASS_COMPUTE);
static_assert(XE_MEM_REGION_CLASS_VRAM == I915_MEMORY_CLASS_DEVICE);
static_assert(XE_MEM_REGION_CLASS_SYSMEM == I915_MEMORY_CLASS_SYSTEM);
int IoctlHelperXe::xeGetQuery(Query *data) {
if (data->numItems == 1) {
QueryItem *queryItem = (QueryItem *)data->itemsPtr;
std::vector<uint8_t> *queryData = nullptr;
switch (queryItem->queryId) {
case static_cast<int>(DrmParam::QueryHwconfigTable):
queryData = &hwconfigFakei915;
break;
default:
xeLog("error: bad query 0x%x\n", queryItem->queryId);
return -1;
}
if (queryData != nullptr) {
if (queryItem->length == 0) {
queryItem->length = static_cast<int32_t>(queryData->size());
return 0;
}
if (queryItem->length != static_cast<int32_t>(queryData->size())) {
xeLog("error: incorrect length 0x%x 0x%lx\n", queryItem->length, queryData->size());
return -1;
}
memcpy_s(reinterpret_cast<void *>(queryItem->dataPtr),
queryItem->length, queryData->data(), queryItem->length);
return 0;
}
}
return -1;
}
const char *IoctlHelperXe::xeGetClassName(int className) {
switch (className) {
case DRM_XE_ENGINE_CLASS_RENDER:
return "rcs";
case DRM_XE_ENGINE_CLASS_COPY:
return "bcs";
case DRM_XE_ENGINE_CLASS_VIDEO_DECODE:
return "vcs";
case DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE:
return "vecs";
case DRM_XE_ENGINE_CLASS_COMPUTE:
return "ccs";
}
return "???";
}
const char *IoctlHelperXe::xeGetBindOpName(int bindOp) {
switch (bindOp) {
case XE_VM_BIND_OP_MAP:
return "MAP";
case XE_VM_BIND_OP_UNMAP:
return "UNMAP";
case XE_VM_BIND_OP_MAP_USERPTR:
return "MAP_USERPTR";
case XE_VM_BIND_OP_MAP | XE_VM_BIND_FLAG_ASYNC:
return "AS_MAP";
case XE_VM_BIND_OP_UNMAP | XE_VM_BIND_FLAG_ASYNC:
return "AS_UNMAP";
case XE_VM_BIND_OP_MAP_USERPTR | XE_VM_BIND_FLAG_ASYNC:
return "AS_MAP_USERPTR";
}
return "unknown_OP";
}
const char *IoctlHelperXe::xeGetengineClassName(uint32_t engineClass) {
switch (engineClass) {
case DRM_XE_ENGINE_CLASS_RENDER:
return "DRM_XE_ENGINE_CLASS_RENDER";
case DRM_XE_ENGINE_CLASS_COPY:
return "DRM_XE_ENGINE_CLASS_COPY";
case DRM_XE_ENGINE_CLASS_VIDEO_DECODE:
return "DRM_XE_ENGINE_CLASS_VIDEO_DECODE";
case DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE:
return "DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE";
case DRM_XE_ENGINE_CLASS_COMPUTE:
return "DRM_XE_ENGINE_CLASS_COMPUTE";
default:
return "?";
}
}
IoctlHelperXe::IoctlHelperXe(Drm &drmArg) : IoctlHelper(drmArg) {
xeLog("IoctlHelperXe::IoctlHelperXe\n", "");
}
bool IoctlHelperXe::initialize() {
xeLog("IoctlHelperXe::initialize\n", "");
DebugManager.flags.ForceUserptrAlignment.set(64);
DebugManager.flags.UseVmBind.set(1);
DebugManager.flags.EnableImmediateVmBindExt.set(1);
DebugManager.flags.RenderCompressedBuffersEnabled.set(0);
struct drm_xe_device_query queryConfig = {};
queryConfig.query = DRM_XE_DEVICE_QUERY_CONFIG;
auto retVal = IoctlHelper::ioctl(DrmIoctl::Query, &queryConfig);
if (retVal != 0 || queryConfig.size == 0) {
return false;
}
auto data = std::vector<uint8_t>(sizeof(drm_xe_query_config) + sizeof(uint64_t) * queryConfig.size, 0);
struct drm_xe_query_config *config = reinterpret_cast<struct drm_xe_query_config *>(data.data());
queryConfig.data = castToUint64(config);
IoctlHelper::ioctl(DrmIoctl::Query, &queryConfig);
xeLog("XE_QUERY_CONFIG_REV_AND_DEVICE_ID\t%#llx\n",
config->info[XE_QUERY_CONFIG_REV_AND_DEVICE_ID]);
xeLog(" REV_ID\t\t\t\t%#llx\n",
config->info[XE_QUERY_CONFIG_REV_AND_DEVICE_ID] >> 16);
xeLog(" DEVICE_ID\t\t\t\t%#llx\n",
config->info[XE_QUERY_CONFIG_REV_AND_DEVICE_ID] & 0xffff);
xeLog("XE_QUERY_CONFIG_FLAGS\t\t\t%#llx\n",
config->info[XE_QUERY_CONFIG_FLAGS]);
xeLog(" XE_QUERY_CONFIG_FLAGS_HAS_VRAM\t%s\n",
config->info[XE_QUERY_CONFIG_FLAGS] &
XE_QUERY_CONFIG_FLAGS_HAS_VRAM
? "ON"
: "OFF");
xeLog(" XE_QUERY_CONFIG_FLAGS_USE_GUC\t\t%s\n",
config->info[XE_QUERY_CONFIG_FLAGS] &
XE_QUERY_CONFIG_FLAGS_USE_GUC
? "ON"
: "OFF");
xeLog("XE_QUERY_CONFIG_MIN_ALIGNEMENT\t\t%#llx\n",
config->info[XE_QUERY_CONFIG_MIN_ALIGNEMENT]);
xeLog("XE_QUERY_CONFIG_VA_BITS\t\t%#llx\n",
config->info[XE_QUERY_CONFIG_VA_BITS]);
xeLog("XE_QUERY_CONFIG_GT_COUNT\t\t%llu\n",
config->info[XE_QUERY_CONFIG_GT_COUNT]);
xeLog("XE_QUERY_CONFIG_MEM_REGION_COUNT\t%llu\n",
config->info[XE_QUERY_CONFIG_MEM_REGION_COUNT]);
chipsetId = config->info[XE_QUERY_CONFIG_REV_AND_DEVICE_ID] & 0xffff;
revId = static_cast<int>(config->info[XE_QUERY_CONFIG_REV_AND_DEVICE_ID] >> 16);
hasVram = config->info[XE_QUERY_CONFIG_FLAGS] & XE_QUERY_CONFIG_FLAGS_HAS_VRAM ? 1 : 0;
addressWidth = static_cast<uint32_t>(config->info[XE_QUERY_CONFIG_VA_BITS]);
memset(&queryConfig, 0, sizeof(queryConfig));
queryConfig.query = DRM_XE_DEVICE_QUERY_HWCONFIG;
IoctlHelper::ioctl(DrmIoctl::Query, &queryConfig);
hwconfigFakei915.resize(queryConfig.size);
queryConfig.data = castToUint64(hwconfigFakei915.data());
IoctlHelper::ioctl(DrmIoctl::Query, &queryConfig);
auto hwInfo = this->drm.getRootDeviceEnvironment().getMutableHardwareInfo();
hwInfo->platform.usDeviceID = chipsetId;
hwInfo->platform.usRevId = revId;
return true;
}
IoctlHelperXe::~IoctlHelperXe() {
xeLog("IoctlHelperXe::~IoctlHelperXe\n", "");
}
bool IoctlHelperXe::isSetPairAvailable() {
return false;
}
bool IoctlHelperXe::isChunkingAvailable() {
return false;
}
bool IoctlHelperXe::isVmBindAvailable() {
return true;
}
std::vector<uint8_t> IoctlHelperXe::queryData(uint32_t queryId) {
struct drm_xe_device_query deviceQuery = {};
deviceQuery.query = queryId;
IoctlHelper::ioctl(DrmIoctl::Query, &deviceQuery);
std::vector<uint8_t> retVal(deviceQuery.size);
deviceQuery.data = castToUint64(retVal.data());
IoctlHelper::ioctl(DrmIoctl::Query, &deviceQuery);
return retVal;
}
std::unique_ptr<EngineInfo> IoctlHelperXe::createEngineInfo(bool isSysmanEnabled) {
auto enginesData = queryData(DRM_XE_DEVICE_QUERY_ENGINES);
auto numberHwEngines = enginesData.size() /
sizeof(struct drm_xe_engine_class_instance);
xeLog("numberHwEngines=%d\n", numberHwEngines);
if (enginesData.empty()) {
return {};
}
auto queriedEngines = reinterpret_cast<struct drm_xe_engine_class_instance *>(enginesData.data());
StackVec<std::vector<EngineClassInstance>, 2> enginesPerTile{};
std::bitset<8> multiTileMask{};
for (auto i = 0u; i < numberHwEngines; i++) {
auto tile = queriedEngines[i].gt_id;
multiTileMask.set(tile);
EngineClassInstance engineClassInstance{};
engineClassInstance.engineClass = queriedEngines[i].engine_class;
engineClassInstance.engineInstance = queriedEngines[i].engine_instance;
xeLog("\t%s:%d\n", xeGetClassName(engineClassInstance.engineClass), engineClassInstance.engineInstance);
if (engineClassInstance.engineClass == getDrmParamValue(DrmParam::EngineClassCompute) ||
engineClassInstance.engineClass == getDrmParamValue(DrmParam::EngineClassRender) ||
engineClassInstance.engineClass == getDrmParamValue(DrmParam::EngineClassCopy) ||
(isSysmanEnabled && (engineClassInstance.engineClass == getDrmParamValue(DrmParam::EngineClassVideo) ||
engineClassInstance.engineClass == getDrmParamValue(DrmParam::EngineClassVideoEnhance)))) {
if (enginesPerTile.size() <= tile) {
enginesPerTile.resize(tile + 1);
}
enginesPerTile[tile].push_back(engineClassInstance);
allEngines.push_back(queriedEngines[i]);
}
}
auto hwInfo = drm.getRootDeviceEnvironment().getMutableHardwareInfo();
if (hwInfo->featureTable.flags.ftrMultiTileArch) {
auto &multiTileArchInfo = hwInfo->gtSystemInfo.MultiTileArchInfo;
multiTileArchInfo.IsValid = true;
multiTileArchInfo.TileCount = multiTileMask.count();
multiTileArchInfo.TileMask = static_cast<uint8_t>(multiTileMask.to_ulong());
}
return std::make_unique<EngineInfo>(&drm, enginesPerTile);
}
inline MemoryRegion createMemoryRegionFromXeMemRegion(const drm_xe_query_mem_usage::drm_xe_query_mem_region &xeMemRegion) {
MemoryRegion memoryRegion{};
memoryRegion.region.memoryInstance = xeMemRegion.instance;
memoryRegion.region.memoryClass = xeMemRegion.mem_class;
memoryRegion.probedSize = xeMemRegion.total_size;
memoryRegion.unallocatedSize = xeMemRegion.total_size - xeMemRegion.used;
return memoryRegion;
}
std::unique_ptr<MemoryInfo> IoctlHelperXe::createMemoryInfo() {
auto memUsageData = queryData(DRM_XE_DEVICE_QUERY_MEM_USAGE);
auto gtsData = queryData(DRM_XE_DEVICE_QUERY_GTS);
if (memUsageData.empty() || gtsData.empty()) {
return {};
}
MemoryInfo::RegionContainer regionsContainer{};
auto xeMemUsageData = reinterpret_cast<drm_xe_query_mem_usage *>(memUsageData.data());
auto xeGtsData = reinterpret_cast<drm_xe_query_gts *>(gtsData.data());
std::array<drm_xe_query_mem_usage::drm_xe_query_mem_region *, 64> memoryRegionInstances{};
for (auto i = 0u; i < xeMemUsageData->num_regions; i++) {
auto &region = xeMemUsageData->regions[i];
memoryRegionInstances[region.instance] = &region;
if (region.mem_class == XE_MEM_REGION_CLASS_SYSMEM) {
regionsContainer.push_back(createMemoryRegionFromXeMemRegion(region));
}
}
if (regionsContainer.empty()) {
return {};
}
for (auto i = 0u; i < xeGtsData->num_gt; i++) {
uint64_t nativeMemRegions = xeGtsData->gts[i].native_mem_regions;
auto regionIndex = Math::log2(nativeMemRegions);
UNRECOVERABLE_IF(!memoryRegionInstances[regionIndex]);
regionsContainer.push_back(createMemoryRegionFromXeMemRegion(*memoryRegionInstances[regionIndex]));
xeTimestampFrequency = xeGtsData->gts[i].clock_freq;
}
return std::make_unique<MemoryInfo>(regionsContainer, drm);
}
void IoctlHelperXe::getTopologyData(uint32_t nTiles, std::vector<std::bitset<8>> geomDss[2], std::vector<std::bitset<8>> computeDss[2], std::vector<std::bitset<8>> euDss[2], DrmQueryTopologyData &topologyData, bool &isComputeDssEmpty) {
int subSliceCount = 0;
int euPerDss = 0;
for (auto tileId = 0u; tileId < nTiles; tileId++) {
int subSliceCountPerTile = 0;
for (auto byte = 0u; byte < computeDss[tileId].size(); byte++) {
subSliceCountPerTile += computeDss[tileId][byte].count();
}
if (subSliceCountPerTile == 0) {
isComputeDssEmpty = true;
for (auto byte = 0u; byte < geomDss[tileId].size(); byte++) {
subSliceCountPerTile += geomDss[tileId][byte].count();
}
}
int euPerDssPerTile = 0;
for (auto byte = 0u; byte < euDss[tileId].size(); byte++) {
euPerDssPerTile += euDss[tileId][byte].count();
}
// pick smallest config
subSliceCount = (subSliceCount == 0) ? subSliceCountPerTile : std::min(subSliceCount, subSliceCountPerTile);
euPerDss = (euPerDss == 0) ? euPerDssPerTile : std::min(euPerDss, euPerDssPerTile);
// pick max config
topologyData.maxSubSliceCount = std::max(topologyData.maxSubSliceCount, subSliceCountPerTile);
topologyData.maxEuCount = std::max(topologyData.maxEuCount, euPerDssPerTile * subSliceCountPerTile);
}
topologyData.sliceCount = 1;
topologyData.subSliceCount = subSliceCount;
topologyData.euCount = subSliceCount * euPerDss;
topologyData.maxSliceCount = 1;
}
void IoctlHelperXe::getTopologyMap(uint32_t nTiles, std::vector<std::bitset<8>> dssInfo[2], TopologyMap &topologyMap) {
for (auto tileId = 0u; tileId < nTiles; tileId++) {
std::vector<int> sliceIndices;
std::vector<int> subSliceIndices;
sliceIndices.push_back(0);
for (auto byte = 0u; byte < dssInfo[tileId].size(); byte++) {
for (auto bit = 0u; bit < 8u; bit++) {
if (dssInfo[tileId][byte].test(bit)) {
auto subSliceIndex = byte * 8 + bit;
subSliceIndices.push_back(subSliceIndex);
}
}
}
topologyMap[tileId].sliceIndices = std::move(sliceIndices);
topologyMap[tileId].subsliceIndices = std::move(subSliceIndices);
}
}
bool IoctlHelperXe::getTopologyDataAndMap(const HardwareInfo &hwInfo, DrmQueryTopologyData &topologyData, TopologyMap &topologyMap) {
auto queryGtTopology = queryData(DRM_XE_DEVICE_QUERY_GT_TOPOLOGY);
auto fillMask = [](std::vector<std::bitset<8>> &vec, drm_xe_query_topology_mask *topo) {
for (uint32_t j = 0; j < topo->num_bytes; j++) {
vec.push_back(topo->mask[j]);
}
};
std::vector<std::bitset<8>> geomDss[2];
std::vector<std::bitset<8>> computeDss[2];
std::vector<std::bitset<8>> euDss[2];
auto topologySize = queryGtTopology.size();
uint8_t *dataPtr = reinterpret_cast<uint8_t *>(queryGtTopology.data());
auto nTiles = 1u;
while (topologySize >= sizeof(drm_xe_query_topology_mask)) {
drm_xe_query_topology_mask *topo = reinterpret_cast<drm_xe_query_topology_mask *>(dataPtr);
UNRECOVERABLE_IF(topo == nullptr);
uint32_t tileId = topo->gt_id;
nTiles = std::max(tileId + 1, nTiles);
switch (topo->type) {
case XE_TOPO_DSS_GEOMETRY:
fillMask(geomDss[tileId], topo);
break;
case XE_TOPO_DSS_COMPUTE:
fillMask(computeDss[tileId], topo);
break;
case XE_TOPO_EU_PER_DSS:
fillMask(euDss[tileId], topo);
break;
default:
xeLog("Unhandle GT Topo type: %d\n", topo->type);
return false;
}
uint32_t itemSize = sizeof(drm_xe_query_topology_mask) + topo->num_bytes;
topologySize -= itemSize;
dataPtr += itemSize;
}
bool isComputeDssEmpty = false;
getTopologyData(nTiles, geomDss, computeDss, euDss, topologyData, isComputeDssEmpty);
auto &dssInfo = isComputeDssEmpty ? geomDss : computeDss;
getTopologyMap(nTiles, dssInfo, topologyMap);
return true;
}
int IoctlHelperXe::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional<uint32_t> vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) {
struct drm_xe_gem_create create = {};
uint32_t regionsSize = static_cast<uint32_t>(memClassInstances.size());
if (!regionsSize) {
xeLog("memClassInstances empty !\n", "");
return -1;
}
if (vmId != std::nullopt) {
create.vm_id = vmId.value();
}
create.size = allocSize;
MemoryClassInstance mem = memClassInstances[regionsSize - 1];
std::bitset<32> memoryInstances{};
for (const auto &memoryClassInstance : memClassInstances) {
memoryInstances.set(memoryClassInstance.memoryInstance);
}
create.flags = static_cast<uint32_t>(memoryInstances.to_ulong());
auto ret = IoctlHelper::ioctl(DrmIoctl::GemCreate, &create);
handle = create.handle;
xeLog(" -> IoctlHelperXe::%s [%d,%d] vmid=0x%x s=0x%lx f=0x%x h=0x%x r=%d\n", __FUNCTION__,
mem.memoryClass, mem.memoryInstance,
create.vm_id, create.size, create.flags, handle, ret);
{
std::unique_lock<std::mutex> lock(xeLock);
BindInfo b = {create.handle, 0, 0, create.size};
bindInfo.push_back(b);
}
return ret;
}
CacheRegion IoctlHelperXe::closAlloc() {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return CacheRegion::None;
}
uint16_t IoctlHelperXe::closAllocWays(CacheRegion closIndex, uint16_t cacheLevel, uint16_t numWays) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
}
CacheRegion IoctlHelperXe::closFree(CacheRegion closIndex) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return CacheRegion::None;
}
int IoctlHelperXe::xeWaitUserFence(uint64_t mask, uint16_t op, uint64_t addr, uint64_t value,
int64_t timeout) {
struct drm_xe_wait_user_fence wait = {};
wait.addr = addr;
wait.op = op;
wait.flags = DRM_XE_UFENCE_WAIT_SOFT_OP;
wait.value = value;
wait.mask = mask;
wait.timeout = timeout;
wait.num_engines = 0;
wait.instances = 0;
auto retVal = IoctlHelper::ioctl(DrmIoctl::GemWaitUserFence, &wait);
xeLog(" -> IoctlHelperXe::%s a=0x%llx v=0x%llx T=0x%llx F=0x%x retVal=0x%x\n", __FUNCTION__, addr, value,
timeout, wait.flags, retVal);
return retVal;
}
int IoctlHelperXe::waitUserFence(uint32_t ctxId, uint64_t address,
uint64_t value, uint32_t dataWidth, int64_t timeout, uint16_t flags) {
xeLog(" -> IoctlHelperXe::%s a=0x%llx v=0x%llx w=0x%x T=0x%llx F=0x%x\n", __FUNCTION__, address, value, dataWidth, timeout, flags);
uint64_t mask;
switch (dataWidth) {
case static_cast<uint32_t>(Drm::ValueWidth::U64):
mask = DRM_XE_UFENCE_WAIT_U64;
break;
case static_cast<uint32_t>(Drm::ValueWidth::U32):
mask = DRM_XE_UFENCE_WAIT_U32;
break;
case static_cast<uint32_t>(Drm::ValueWidth::U16):
mask = DRM_XE_UFENCE_WAIT_U16;
break;
default:
mask = DRM_XE_UFENCE_WAIT_U8;
break;
}
if (timeout == -1) {
/* expected in i915 but not in xe where timeout is an unsigned long */
timeout = TimeoutControls::maxTimeout;
}
if (address) {
return xeWaitUserFence(mask, DRM_XE_UFENCE_WAIT_GTE, address, value, timeout);
}
return 0;
}
uint32_t IoctlHelperXe::getAtomicAdvise(bool isNonAtomic) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
}
uint32_t IoctlHelperXe::getPreferredLocationAdvise() {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
}
std::optional<MemoryClassInstance> IoctlHelperXe::getPreferredLocationRegion(PreferredLocation memoryLocation, uint32_t memoryInstance) {
return std::nullopt;
}
bool IoctlHelperXe::setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return false;
}
bool IoctlHelperXe::setVmBoAdviseForChunking(int32_t handle, uint64_t start, uint64_t length, uint32_t attribute, void *region) {
return false;
}
bool IoctlHelperXe::setVmPrefetch(uint64_t start, uint64_t length, uint32_t region, uint32_t vmId) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return false;
}
uint32_t IoctlHelperXe::getDirectSubmissionFlag() {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
}
uint16_t IoctlHelperXe::getWaitUserFenceSoftFlag() {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
};
int IoctlHelperXe::execBuffer(ExecBuffer *execBuffer, uint64_t completionGpuAddress, TaskCountType counterValue) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
int ret = 0;
if (execBuffer) {
drm_i915_gem_execbuffer2 *d = reinterpret_cast<drm_i915_gem_execbuffer2 *>(execBuffer->data);
if (d) {
drm_i915_gem_exec_object2 *obj = reinterpret_cast<drm_i915_gem_exec_object2
*>(d->buffers_ptr);
uint32_t engine = static_cast<uint32_t>(d->rsvd1);
if (obj) {
xeLog("EXEC bc=%d ofs=%d len=%d f=0x%llx ctx=0x%x ptr=0x%llx r=0x%x\n",
d->buffer_count, d->batch_start_offset, d->batch_len,
d->flags, engine, obj->offset, ret);
xeLog(" -> IoctlHelperXe::%s CA=0x%llx v=0x%x ctx=0x%x\n", __FUNCTION__,
completionGpuAddress, counterValue, engine);
struct drm_xe_sync sync[1] = {};
sync[0].flags = DRM_XE_SYNC_USER_FENCE | DRM_XE_SYNC_SIGNAL;
sync[0].addr = completionGpuAddress;
sync[0].timeline_value = counterValue;
struct drm_xe_exec exec = {};
exec.engine_id = engine;
exec.num_syncs = 1;
exec.syncs = reinterpret_cast<uintptr_t>(&sync);
exec.address = obj->offset + d->batch_start_offset;
exec.num_batch_buffer = 1;
ret = IoctlHelper::ioctl(DrmIoctl::GemExecbuffer2, &exec);
xeLog("r=0x%x batch=0x%lx\n", ret, exec.address);
if (DebugManager.flags.PrintCompletionFenceUsage.get()) {
std::cout << "Completion fence submitted."
<< " GPU address: " << std::hex << completionGpuAddress << std::dec
<< ", value: " << counterValue << std::endl;
}
}
}
}
return ret;
}
bool IoctlHelperXe::completionFenceExtensionSupported(const bool isVmBindAvailable) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return isVmBindAvailable;
}
std::unique_ptr<uint8_t[]> IoctlHelperXe::prepareVmBindExt(const StackVec<uint32_t, 2> &bindExtHandles) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return {};
}
uint64_t IoctlHelperXe::getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident) {
uint64_t ret = 0;
xeLog(" -> IoctlHelperXe::%s %d %d %d\n", __FUNCTION__, bindCapture, bindImmediate, bindMakeResident);
if (bindCapture) {
ret |= XE_NEO_BIND_CAPTURE_FLAG;
}
if (bindImmediate) {
ret |= XE_NEO_BIND_IMMEDIATE_FLAG;
}
if (bindMakeResident) {
ret |= XE_NEO_BIND_MAKERESIDENT_FLAG;
}
return ret;
}
int IoctlHelperXe::queryDistances(std::vector<QueryItem> &queryItems, std::vector<DistanceInfo> &distanceInfos) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
}
std::optional<DrmParam> IoctlHelperXe::getHasPageFaultParamId() {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return {};
};
bool IoctlHelperXe::getEuStallProperties(std::array<uint64_t, 12u> &properties, uint64_t dssBufferSize, uint64_t samplingRate,
uint64_t pollPeriod, uint64_t engineInstance, uint64_t notifyNReports) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return false;
}
uint32_t IoctlHelperXe::getEuStallFdParameter() {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
}
std::unique_ptr<uint8_t[]> IoctlHelperXe::createVmControlExtRegion(const std::optional<MemoryClassInstance> &regionInstanceClass) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return {};
}
uint32_t IoctlHelperXe::getFlagsForVmCreate(bool disableScratch, bool enablePageFault, bool useVmBind) {
xeLog(" -> IoctlHelperXe::%s %d,%d,%d\n", __FUNCTION__, disableScratch, enablePageFault, useVmBind);
uint32_t flags = 0u;
if (disableScratch) {
flags |= XE_NEO_VMCREATE_DISABLESCRATCH_FLAG;
}
if (enablePageFault) {
flags |= XE_NEO_VMCREATE_ENABLEPAGEFAULT_FLAG;
}
if (useVmBind) {
flags |= XE_NEO_VMCREATE_USEVMBIND_FLAG;
}
return flags;
}
uint32_t IoctlHelperXe::createContextWithAccessCounters(GemContextCreateExt &gcc) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
}
uint32_t IoctlHelperXe::createCooperativeContext(GemContextCreateExt &gcc) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
}
void IoctlHelperXe::fillVmBindExtSetPat(VmBindExtSetPatT &vmBindExtSetPat, uint64_t patIndex, uint64_t nextExtension) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
}
void IoctlHelperXe::fillVmBindExtUserFence(VmBindExtUserFenceT &vmBindExtUserFence, uint64_t fenceAddress, uint64_t fenceValue, uint64_t nextExtension) {
xeLog(" -> IoctlHelperXe::%s 0x%lx 0x%lx\n", __FUNCTION__, fenceAddress, fenceValue);
auto xeBindExtUserFence = reinterpret_cast<UserFenceExtension *>(vmBindExtUserFence);
UNRECOVERABLE_IF(!xeBindExtUserFence);
xeBindExtUserFence->tag = UserFenceExtension::tagValue;
xeBindExtUserFence->addr = fenceAddress;
xeBindExtUserFence->value = fenceValue;
}
std::optional<uint64_t> IoctlHelperXe::getCopyClassSaturatePCIECapability() {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return {};
}
std::optional<uint64_t> IoctlHelperXe::getCopyClassSaturateLinkCapability() {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return {};
}
uint32_t IoctlHelperXe::getVmAdviseAtomicAttribute() {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
}
int IoctlHelperXe::vmBind(const VmBindParams &vmBindParams) {
return xeVmBind(vmBindParams, true);
}
int IoctlHelperXe::vmUnbind(const VmBindParams &vmBindParams) {
return xeVmBind(vmBindParams, false);
}
UuidRegisterResult IoctlHelperXe::registerUuid(const std::string &uuid, uint32_t uuidClass, uint64_t ptr, uint64_t size) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return {};
}
UuidRegisterResult IoctlHelperXe::registerStringClassUuid(const std::string &uuid, uint64_t ptr, uint64_t size) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return {};
}
int IoctlHelperXe::unregisterUuid(uint32_t handle) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
}
bool IoctlHelperXe::isContextDebugSupported() {
return false;
}
int IoctlHelperXe::setContextDebugFlag(uint32_t drmContextId) {
return 0;
}
bool IoctlHelperXe::isDebugAttachAvailable() {
return false;
}
unsigned int IoctlHelperXe::getIoctlRequestValue(DrmIoctl ioctlRequest) const {
xeLog(" -> IoctlHelperXe::%s 0x%x\n", __FUNCTION__, ioctlRequest);
switch (ioctlRequest) {
case DrmIoctl::SyncobjCreate:
RETURN_ME(DRM_IOCTL_SYNCOBJ_CREATE);
case DrmIoctl::SyncobjWait:
RETURN_ME(DRM_IOCTL_SYNCOBJ_WAIT);
case DrmIoctl::SyncobjDestroy:
RETURN_ME(DRM_IOCTL_SYNCOBJ_DESTROY);
case DrmIoctl::GemClose:
RETURN_ME(DRM_IOCTL_GEM_CLOSE);
case DrmIoctl::GemVmCreate:
RETURN_ME(DRM_IOCTL_XE_VM_CREATE);
case DrmIoctl::GemVmDestroy:
RETURN_ME(DRM_IOCTL_XE_VM_DESTROY);
case DrmIoctl::GemMmapOffset:
RETURN_ME(DRM_IOCTL_XE_GEM_MMAP_OFFSET);
case DrmIoctl::GemCreate:
RETURN_ME(DRM_IOCTL_XE_GEM_CREATE);
case DrmIoctl::GemExecbuffer2:
RETURN_ME(DRM_IOCTL_XE_EXEC);
case DrmIoctl::GemVmBind:
RETURN_ME(DRM_IOCTL_XE_VM_BIND);
case DrmIoctl::Query:
RETURN_ME(DRM_IOCTL_XE_DEVICE_QUERY);
case DrmIoctl::GemContextCreateExt:
RETURN_ME(DRM_IOCTL_XE_ENGINE_CREATE);
case DrmIoctl::GemContextDestroy:
RETURN_ME(DRM_IOCTL_XE_ENGINE_DESTROY);
case DrmIoctl::GemWaitUserFence:
RETURN_ME(DRM_IOCTL_XE_WAIT_USER_FENCE);
case DrmIoctl::PrimeFdToHandle:
RETURN_ME(DRM_IOCTL_PRIME_FD_TO_HANDLE);
case DrmIoctl::PrimeHandleToFd:
RETURN_ME(DRM_IOCTL_PRIME_HANDLE_TO_FD);
case DrmIoctl::RegRead:
RETURN_ME(DRM_IOCTL_XE_MMIO);
default:
UNRECOVERABLE_IF(true);
return 0;
}
}
int IoctlHelperXe::getDrmParamValue(DrmParam drmParam) const {
xeLog(" -> IoctlHelperXe::%s 0x%x %s\n", __FUNCTION__, drmParam, getDrmParamString(drmParam).c_str());
switch (drmParam) {
case DrmParam::MemoryClassDevice:
return I915_MEMORY_CLASS_DEVICE;
case DrmParam::MemoryClassSystem:
return I915_MEMORY_CLASS_SYSTEM;
case DrmParam::EngineClassRender:
return drm_i915_gem_engine_class::I915_ENGINE_CLASS_RENDER;
case DrmParam::EngineClassCopy:
return drm_i915_gem_engine_class::I915_ENGINE_CLASS_COPY;
case DrmParam::EngineClassVideo:
return drm_i915_gem_engine_class::I915_ENGINE_CLASS_VIDEO;
case DrmParam::EngineClassVideoEnhance:
return drm_i915_gem_engine_class::I915_ENGINE_CLASS_VIDEO_ENHANCE;
case DrmParam::EngineClassCompute:
return prelim_drm_i915_gem_engine_class::PRELIM_I915_ENGINE_CLASS_COMPUTE;
case DrmParam::EngineClassInvalid:
return drm_i915_gem_engine_class::I915_ENGINE_CLASS_INVALID;
default:
return getDrmParamValueBase(drmParam);
}
}
int IoctlHelperXe::getDrmParamValueBase(DrmParam drmParam) const {
return static_cast<int>(drmParam);
}
template <typename... XeLogArgs>
void IoctlHelperXe::xeLog(XeLogArgs &&...args) const {
PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, args...);
}
std::string IoctlHelperXe::getIoctlString(DrmIoctl ioctlRequest) const {
switch (ioctlRequest) {
case DrmIoctl::SyncobjCreate:
STRINGIFY_ME(DRM_IOCTL_SYNCOBJ_CREATE);
case DrmIoctl::SyncobjWait:
STRINGIFY_ME(DRM_IOCTL_SYNCOBJ_WAIT);
case DrmIoctl::SyncobjDestroy:
STRINGIFY_ME(DRM_IOCTL_SYNCOBJ_DESTROY);
case DrmIoctl::GemClose:
STRINGIFY_ME(DRM_IOCTL_GEM_CLOSE);
case DrmIoctl::GemVmCreate:
STRINGIFY_ME(DRM_IOCTL_XE_VM_CREATE);
case DrmIoctl::GemVmDestroy:
STRINGIFY_ME(DRM_IOCTL_XE_VM_DESTROY);
case DrmIoctl::GemMmapOffset:
STRINGIFY_ME(DRM_IOCTL_XE_GEM_MMAP_OFFSET);
case DrmIoctl::GemCreate:
STRINGIFY_ME(DRM_IOCTL_XE_GEM_CREATE);
case DrmIoctl::GemExecbuffer2:
STRINGIFY_ME(DRM_IOCTL_XE_EXEC);
case DrmIoctl::GemVmBind:
STRINGIFY_ME(DRM_IOCTL_XE_VM_BIND);
case DrmIoctl::Query:
STRINGIFY_ME(DRM_IOCTL_XE_DEVICE_QUERY);
case DrmIoctl::GemContextCreateExt:
STRINGIFY_ME(DRM_IOCTL_XE_ENGINE_CREATE);
case DrmIoctl::GemContextDestroy:
STRINGIFY_ME(DRM_IOCTL_XE_ENGINE_DESTROY);
case DrmIoctl::GemWaitUserFence:
STRINGIFY_ME(DRM_IOCTL_XE_WAIT_USER_FENCE);
case DrmIoctl::PrimeFdToHandle:
STRINGIFY_ME(DRM_IOCTL_PRIME_FD_TO_HANDLE);
case DrmIoctl::PrimeHandleToFd:
STRINGIFY_ME(DRM_IOCTL_PRIME_HANDLE_TO_FD);
case DrmIoctl::RegRead:
STRINGIFY_ME(DRM_IOCTL_XE_MMIO);
default:
return "???";
}
}
uint32_t IoctlHelperXe::xeSyncObjCreate(uint32_t flags) {
int ret;
struct drm_syncobj_create create = {};
create.flags = flags;
ret = IoctlHelper::ioctl(DrmIoctl::SyncobjCreate, &create);
UNRECOVERABLE_IF(ret);
return create.handle;
}
bool IoctlHelperXe::xeSyncObjWait(uint32_t *handles, uint32_t count,
uint64_t absTimeoutNsec, uint32_t flags,
uint32_t *firstSignaled) {
int ret;
struct drm_syncobj_wait wait = {};
wait.handles = castToUint64(handles);
wait.timeout_nsec = absTimeoutNsec;
wait.count_handles = count;
wait.flags = flags;
ret = IoctlHelper::ioctl(DrmIoctl::SyncobjWait, &wait);
if (ret && errno == ETIME) {
return false;
}
UNRECOVERABLE_IF(ret);
if (firstSignaled)
*firstSignaled = wait.first_signaled;
return true;
}
void IoctlHelperXe::xeSyncObjDestroy(uint32_t handle) {
int ret;
struct drm_syncobj_destroy destroy = {};
destroy.handle = handle;
ret = IoctlHelper::ioctl(DrmIoctl::SyncobjDestroy, &destroy);
UNRECOVERABLE_IF(ret);
}
uint64_t IoctlHelperXe::xeDecanonize(uint64_t address) {
return (address & maxNBitValue(addressWidth));
}
int IoctlHelperXe::ioctl(DrmIoctl request, void *arg) {
int ret = -1;
xeLog(" => IoctlHelperXe::%s 0x%x\n", __FUNCTION__, request);
switch (request) {
case DrmIoctl::Getparam: {
struct GetParam *d = (struct GetParam *)arg;
ret = 0;
switch (d->param) {
case static_cast<int>(DrmParam::ParamChipsetId):
*d->value = chipsetId;
break;
case static_cast<int>(DrmParam::ParamRevision):
*d->value = revId;
break;
case static_cast<int>(DrmParam::ParamHasPageFault):
*d->value = 0;
break;
case static_cast<int>(DrmParam::ParamHasExecSoftpin):
*d->value = 1;
break;
case static_cast<int>(DrmParam::ParamHasScheduler):
*d->value = static_cast<int>(0x80000037);
break;
case static_cast<int>(DrmParam::ParamCsTimestampFrequency):
*d->value = static_cast<int>(xeTimestampFrequency);
break;
default:
ret = -1;
}
xeLog(" -> IoctlHelperXe::ioctl Getparam 0x%x/0x%x r=%d\n", d->param, *d->value, ret);
} break;
case DrmIoctl::Query: {
Query *q = static_cast<Query *>(arg);
ret = xeGetQuery(q);
if (ret == 0) {
QueryItem *queryItem = reinterpret_cast<QueryItem *>(q->itemsPtr);
xeLog(" -> IoctlHelperXe::ioctl Query id=0x%x f=0x%x len=%d r=%d\n",
static_cast<int>(queryItem->queryId), static_cast<int>(queryItem->flags), queryItem->length, ret);
} else {
xeLog(" -> IoctlHelperXe::ioctl Query r=%d\n", ret);
}
} break;
case DrmIoctl::GemUserptr: {
GemUserPtr *d = static_cast<GemUserPtr *>(arg);
d->handle = userPtrHandle++ | XE_USERPTR_FAKE_FLAG;
{
std::unique_lock<std::mutex> lock(xeLock);
BindInfo b = {d->handle, d->userPtr, 0, d->userSize};
bindInfo.push_back(b);
}
ret = 0;
xeLog(" -> IoctlHelperXe::ioctl GemUserptrGemUserptr p=0x%llx s=0x%llx f=0x%x h=0x%x r=%d\n", d->userPtr,
d->userSize, d->flags, d->handle, ret);
xeShowBindTable();
} break;
case DrmIoctl::GemContextCreateExt: {
UNRECOVERABLE_IF(true);
} break;
case DrmIoctl::GemContextDestroy: {
GemContextDestroy *d = static_cast<GemContextDestroy *>(arg);
struct drm_xe_engine_destroy destroy = {};
destroy.engine_id = d->contextId;
if (d->contextId != 0xffffffff)
ret = IoctlHelper::ioctl(request, &destroy);
else
ret = 0;
xeLog(" -> IoctlHelperXe::ioctl GemContextDestroryExt ctx=0x%x r=%d\n",
d->contextId, ret);
} break;
case DrmIoctl::GemContextGetparam: {
GemContextParam *d = static_cast<GemContextParam *>(arg);
ret = 0;
switch (d->param) {
case static_cast<int>(DrmParam::ContextParamGttSize):
d->value = 0x1ull << addressWidth;
break;
case static_cast<int>(DrmParam::ContextParamSseu):
d->value = 0x55fdd94d4e40;
break;
case static_cast<int>(DrmParam::ContextParamPersistence):
d->value = 0x1;
break;
default:
ret = -1;
break;
}
xeLog(" -> IoctlHelperXe::ioctl GemContextGetparam r=%d\n", ret);
} break;
case DrmIoctl::GemContextSetparam: {
GemContextParam *d = static_cast<GemContextParam *>(arg);
switch (d->param) {
case static_cast<int>(DrmParam::ContextParamPersistence):
if (d->value == 0)
ret = 0;
break;
case static_cast<int>(DrmParam::ContextParamEngines): {
i915_context_param_engines *contextEngine = reinterpret_cast<i915_context_param_engines *>(d->value);
int items = (d->size - sizeof(uint64_t)) / sizeof(uint32_t);
contextParamEngine.clear();
if (items < 11) {
for (int i = 0; i < items; i++) {
drm_xe_engine_class_instance engine = {
contextEngine->engines[i].engine_class,
contextEngine->engines[i].engine_instance,
0};
if (engine.engine_class != 65535)
contextParamEngine.push_back(engine);
}
}
if (contextParamEngine.size())
ret = 0;
} break;
case contextPrivateParamBoost:
ret = 0;
break;
default:
ret = -1;
break;
}
xeLog(" -> IoctlHelperXe::ioctl GemContextSetparam r=%d\n", ret);
} break;
case DrmIoctl::GemClose: {
struct GemClose *d = static_cast<struct GemClose *>(arg);
int found = -1;
xeShowBindTable();
for (unsigned int i = 0; i < bindInfo.size(); i++) {
if (d->handle == bindInfo[i].handle) {
found = i;
break;
}
}
if (found != -1) {
xeLog(" removing %d: 0x%x 0x%lx 0x%lx\n",
found,
bindInfo[found].handle,
bindInfo[found].userptr,
bindInfo[found].addr);
{
std::unique_lock<std::mutex> lock(xeLock);
bindInfo.erase(bindInfo.begin() + found);
}
if (d->handle & XE_USERPTR_FAKE_FLAG) {
// nothing to do under XE
ret = 0;
} else {
ret = IoctlHelper::ioctl(request, arg);
}
} else {
ret = 0; // let it pass trough for now
}
xeLog(" -> IoctlHelperXe::ioctl GemClose found=%d h=0x%x r=%d\n", found, d->handle, ret);
} break;
case DrmIoctl::RegRead: {
struct drm_xe_mmio mmio = {};
RegisterRead *reg = static_cast<RegisterRead *>(arg);
mmio.addr = static_cast<uint32_t>(reg->offset);
if (reg->offset == (REG_GLOBAL_TIMESTAMP_LDW | 1)) {
mmio.addr = REG_GLOBAL_TIMESTAMP_LDW;
}
mmio.flags = DRM_XE_MMIO_READ | DRM_XE_MMIO_64BIT;
ret = IoctlHelper::ioctl(request, &mmio);
reg->value = mmio.value;
xeLog(" -> IoctlHelperXe::ioctl RegRead 0x%lx/0x%lx r=%d\n",
reg->offset, reg->value, ret);
} break;
case DrmIoctl::GemVmCreate: {
GemVmControl *d = static_cast<GemVmControl *>(arg);
struct drm_xe_vm_create args = {};
args.flags = DRM_XE_VM_CREATE_ASYNC_BIND_OPS |
DRM_XE_VM_CREATE_COMPUTE_MODE;
ret = IoctlHelper::ioctl(request, &args);
d->vmId = ret ? 0 : args.vm_id;
xeVmId = d->vmId;
xeLog(" -> IoctlHelperXe::ioctl GemVmCreate vmid=0x%x r=%d\n", d->vmId, ret);
} break;
case DrmIoctl::GemVmDestroy: {
GemVmControl *d = static_cast<GemVmControl *>(arg);
struct drm_xe_vm_destroy args = {};
args.vm_id = d->vmId;
ret = IoctlHelper::ioctl(request, &args);
xeLog(" -> IoctlHelperXe::ioctl GemVmDestroy vmid=0x%x r=%d\n", d->vmId, ret);
} break;
case DrmIoctl::GemMmapOffset: {
GemMmapOffset *d = static_cast<GemMmapOffset *>(arg);
struct drm_xe_gem_mmap_offset mmo = {};
mmo.handle = d->handle;
ret = IoctlHelper::ioctl(request, &mmo);
d->offset = mmo.offset;
xeLog(" -> IoctlHelperXe::ioctl GemMmapOffset h=0x%x o=0x%x r=%d\n",
d->handle, d->offset, ret);
} break;
case DrmIoctl::GetResetStats: {
ResetStats *d = static_cast<ResetStats *>(arg);
// d->batchActive = 1; // fake gpu hang
ret = 0;
xeLog(" -> IoctlHelperXe::ioctl GetResetStats ctx=0x%x r=%d\n",
d->contextId, ret);
} break;
case DrmIoctl::PrimeFdToHandle: {
PrimeHandle *prime = static_cast<PrimeHandle *>(arg);
ret = IoctlHelper::ioctl(request, arg);
xeLog(" ->PrimeFdToHandle h=0x%x f=0x%x d=0x%x r=%d\n",
prime->handle, prime->flags, prime->fileDescriptor, ret);
} break;
case DrmIoctl::PrimeHandleToFd: {
PrimeHandle *prime = static_cast<PrimeHandle *>(arg);
xeLog(" ->PrimeHandleToFd h=0x%x f=0x%x d=0x%x r=%d\n",
prime->handle, prime->flags, prime->fileDescriptor, ret);
ret = IoctlHelper::ioctl(request, arg);
} break;
case DrmIoctl::GemCreate: {
GemCreate *gemCreate = static_cast<GemCreate *>(arg);
ret = IoctlHelper::ioctl(request, arg);
xeLog(" -> IoctlHelperXe::ioctl GemCreate h=0x%x s=0x%llx r=%d\n",
gemCreate->handle, gemCreate->size, ret);
} break;
default:
xeLog("Not handled 0x%x\n", request);
UNRECOVERABLE_IF(true);
}
return ret;
}
void IoctlHelperXe::xeShowBindTable() {
#if 1
std::unique_lock<std::mutex> lock(xeLock);
xeLog("show bind: (<index> <handle> <userptr> <addr> <size>)\n", "");
for (unsigned int i = 0; i < bindInfo.size(); i++) {
xeLog(" %3d x%08x x%016lx x%016lx x%016lx\n", i,
bindInfo[i].handle,
bindInfo[i].userptr,
bindInfo[i].addr,
bindInfo[i].size);
}
#endif
}
int IoctlHelperXe::createDrmContext(Drm &drm, OsContextLinux &osContext, uint32_t drmVmId, uint32_t deviceIndex) {
struct drm_xe_engine_create create = {};
uint32_t drmContextId = 0;
struct drm_xe_engine_class_instance *currentEngine = nullptr;
std::vector<struct drm_xe_engine_class_instance> engine;
int requestClass = 0;
xeLog("createDrmContext VM=0x%x\n", drmVmId);
auto engineFlag = drm.bindDrmContext(drmContextId, deviceIndex, osContext.getEngineType(), osContext.isEngineInstanced());
switch (engineFlag) {
case static_cast<int>(DrmParam::ExecRender):
requestClass = DRM_XE_ENGINE_CLASS_RENDER;
break;
case static_cast<int>(DrmParam::ExecBlt):
requestClass = DRM_XE_ENGINE_CLASS_COPY;
break;
case static_cast<int>(DrmParam::ExecDefault):
requestClass = DRM_XE_ENGINE_CLASS_COMPUTE;
break;
default:
xeLog("unexpected engineFlag=0x%x\n", engineFlag);
UNRECOVERABLE_IF(true);
}
size_t n = contextParamEngine.size();
create.vm_id = drmVmId;
create.width = 1;
if (n == 0) {
currentEngine = xeFindMatchingEngine(requestClass, XE_FIND_INVALID_INSTANCE);
if (currentEngine == nullptr) {
xeLog("Unable to find engine %d\n", requestClass);
UNRECOVERABLE_IF(true);
return 0;
}
engine.push_back(*currentEngine);
} else {
for (size_t i = 0; i < n; i++) {
currentEngine = xeFindMatchingEngine(contextParamEngine[i].engine_class,
contextParamEngine[i].engine_instance);
if (currentEngine == nullptr) {
xeLog("Unable to find engine %d:%d\n",
contextParamEngine[i].engine_class,
contextParamEngine[i].engine_instance);
UNRECOVERABLE_IF(true);
return 0;
}
engine.push_back(*currentEngine);
}
}
if (engine.size() > 9) {
xeLog("Too much instances...\n", "");
UNRECOVERABLE_IF(true);
return 0;
}
create.instances = castToUint64(engine.data());
create.num_placements = engine.size();
struct drm_xe_ext_engine_set_property ext = {};
ext.base.name = XE_ENGINE_EXTENSION_SET_PROPERTY;
ext.property = XE_ENGINE_PROPERTY_COMPUTE_MODE;
ext.value = 1;
create.extensions = castToUint64(&ext);
int ret = IoctlHelper::ioctl(DrmIoctl::GemContextCreateExt, &create);
drmContextId = create.engine_id;
xeLog("%s:%d (%d) vmid=0x%x ctx=0x%x r=0x%x\n", xeGetClassName(engine[0].engine_class),
engine[0].engine_instance, create.num_placements, drmVmId, drmContextId, ret);
if (ret != 0) {
UNRECOVERABLE_IF(true);
}
return drmContextId;
}
int IoctlHelperXe::xeVmBind(const VmBindParams &vmBindParams, bool bindOp) {
int ret = -1;
const char *operation = "unbind";
if (bindOp) {
operation = "bind";
}
int found = -1;
if (bindOp) {
for (unsigned int i = 0; i < bindInfo.size(); i++) {
if (vmBindParams.handle == bindInfo[i].handle) {
found = i;
break;
}
}
} else {
uint64_t ad = xeDecanonize(vmBindParams.start);
for (unsigned int i = 0; i < bindInfo.size(); i++) {
if (ad == bindInfo[i].addr) {
found = i;
break;
}
}
}
if (found != -1) {
uint32_t extraBindFlag = 0;
struct drm_xe_sync sync[1] = {};
sync[0].flags = DRM_XE_SYNC_USER_FENCE | DRM_XE_SYNC_SIGNAL;
extraBindFlag = XE_VM_BIND_FLAG_ASYNC;
auto xeBindExtUserFence = reinterpret_cast<UserFenceExtension *>(vmBindParams.extensions);
UNRECOVERABLE_IF(!xeBindExtUserFence);
UNRECOVERABLE_IF(xeBindExtUserFence->tag != UserFenceExtension::tagValue);
sync[0].addr = xeBindExtUserFence->addr;
sync[0].timeline_value = xeBindExtUserFence->value;
struct drm_xe_vm_bind bind = {};
bind.vm_id = vmBindParams.vmId;
bind.num_binds = 1;
bind.bind.obj = vmBindParams.handle;
bind.bind.obj_offset = vmBindParams.offset;
bind.bind.range = vmBindParams.length;
bind.bind.addr = xeDecanonize(vmBindParams.start);
bind.bind.op = XE_VM_BIND_OP_MAP;
bind.num_syncs = 1;
bind.syncs = reinterpret_cast<uintptr_t>(&sync);
if (vmBindParams.handle & XE_USERPTR_FAKE_FLAG) {
bind.bind.obj = 0;
bind.bind.obj_offset = bindInfo[found].userptr;
bind.bind.op = XE_VM_BIND_OP_MAP_USERPTR;
}
if (!bindOp) {
bind.bind.op = XE_VM_BIND_OP_UNMAP;
bind.bind.obj = 0;
if (bindInfo[found].handle & XE_USERPTR_FAKE_FLAG) {
bind.bind.obj_offset = bindInfo[found].userptr;
}
}
bind.bind.op |= extraBindFlag;
bindInfo[found].addr = bind.bind.addr;
xeLog(" vm=%d obj=0x%x off=0x%llx range=0x%llx addr=0x%llx op=%d(%s) nsy=%d\n",
bind.vm_id,
bind.bind.obj,
bind.bind.obj_offset,
bind.bind.range,
bind.bind.addr,
bind.bind.op,
xeGetBindOpName(bind.bind.op),
bind.num_syncs);
ret = IoctlHelper::ioctl(DrmIoctl::GemVmBind, &bind);
if (ret != 0) {
return ret;
}
return xeWaitUserFence(DRM_XE_UFENCE_WAIT_U64, DRM_XE_UFENCE_WAIT_EQ,
sync[0].addr,
sync[0].timeline_value, XE_ONE_SEC);
}
xeLog(" -> IoctlHelperXe::%s %s found=%d vmid=0x%x h=0x%x s=0x%llx o=0x%llx l=0x%llx f=0x%llx r=%d\n",
__FUNCTION__, operation, found, vmBindParams.vmId,
vmBindParams.handle, vmBindParams.start, vmBindParams.offset,
vmBindParams.length, vmBindParams.flags, ret);
return ret;
}
std::string IoctlHelperXe::getDrmParamString(DrmParam drmParam) const {
switch (drmParam) {
case DrmParam::ContextCreateExtSetparam:
return "ContextCreateExtSetparam";
case DrmParam::ContextCreateFlagsUseExtensions:
return "ContextCreateFlagsUseExtensions";
case DrmParam::ContextEnginesExtLoadBalance:
return "ContextEnginesExtLoadBalance";
case DrmParam::ContextParamEngines:
return "ContextParamEngines";
case DrmParam::ContextParamGttSize:
return "ContextParamGttSize";
case DrmParam::ContextParamPersistence:
return "ContextParamPersistence";
case DrmParam::ContextParamPriority:
return "ContextParamPriority";
case DrmParam::ContextParamRecoverable:
return "ContextParamRecoverable";
case DrmParam::ContextParamSseu:
return "ContextParamSseu";
case DrmParam::ContextParamVm:
return "ContextParamVm";
case DrmParam::EngineClassRender:
return "EngineClassRender";
case DrmParam::EngineClassCompute:
return "EngineClassCompute";
case DrmParam::EngineClassCopy:
return "EngineClassCopy";
case DrmParam::EngineClassVideo:
return "EngineClassVideo";
case DrmParam::EngineClassVideoEnhance:
return "EngineClassVideoEnhance";
case DrmParam::EngineClassInvalid:
return "EngineClassInvalid";
case DrmParam::EngineClassInvalidNone:
return "EngineClassInvalidNone";
case DrmParam::ExecBlt:
return "ExecBlt";
case DrmParam::ExecDefault:
return "ExecDefault";
case DrmParam::ExecNoReloc:
return "ExecNoReloc";
case DrmParam::ExecRender:
return "ExecRender";
case DrmParam::MemoryClassDevice:
return "MemoryClassDevice";
case DrmParam::MemoryClassSystem:
return "MemoryClassSystem";
case DrmParam::MmapOffsetWb:
return "MmapOffsetWb";
case DrmParam::MmapOffsetWc:
return "MmapOffsetWc";
case DrmParam::ParamChipsetId:
return "ParamChipsetId";
case DrmParam::ParamRevision:
return "ParamRevision";
case DrmParam::ParamHasExecSoftpin:
return "ParamHasExecSoftpin";
case DrmParam::ParamHasPooledEu:
return "ParamHasPooledEu";
case DrmParam::ParamHasScheduler:
return "ParamHasScheduler";
case DrmParam::ParamEuTotal:
return "ParamEuTotal";
case DrmParam::ParamSubsliceTotal:
return "ParamSubsliceTotal";
case DrmParam::ParamMinEuInPool:
return "ParamMinEuInPool";
case DrmParam::ParamCsTimestampFrequency:
return "ParamCsTimestampFrequency";
case DrmParam::ParamHasVmBind:
return "ParamHasVmBind";
case DrmParam::ParamHasPageFault:
return "ParamHasPageFault";
case DrmParam::QueryEngineInfo:
return "QueryEngineInfo";
case DrmParam::QueryHwconfigTable:
return "QueryHwconfigTable";
case DrmParam::QueryComputeSlices:
return "QueryComputeSlices";
case DrmParam::QueryMemoryRegions:
return "QueryMemoryRegions";
case DrmParam::QueryTopologyInfo:
return "QueryTopologyInfo";
case DrmParam::SchedulerCapPreemption:
return "SchedulerCapPreemption";
case DrmParam::TilingNone:
return "TilingNone";
case DrmParam::TilingY:
return "TilingY";
default:
return "DrmParam::<missing>";
}
}
std::string IoctlHelperXe::getFileForMaxGpuFrequency() const {
return "/device/gt0/freq_max";
}
std::string IoctlHelperXe::getFileForMaxGpuFrequencyOfSubDevice(int subDeviceId) const {
return "/device/gt" + std::to_string(subDeviceId) + "/freq_max";
}
std::string IoctlHelperXe::getFileForMaxMemoryFrequencyOfSubDevice(int subDeviceId) const {
return "/device/gt" + std::to_string(subDeviceId) + "/freq_rp0";
}
std::string IoctlHelperXe::getFileForMemoryAddrRange(int subDeviceId) const {
return "device/tile" + std::to_string(subDeviceId) + "/addr_range";
}
struct drm_xe_engine_class_instance *
IoctlHelperXe::xeFindMatchingEngine(uint16_t engineClass, uint16_t engineInstance) {
for (auto &engine : allEngines) {
if (engine.engine_class == engineClass &&
(engineInstance == XE_FIND_INVALID_INSTANCE || engine.engine_instance == engineInstance)) {
xeLog("\t select: %s:%d (%d)\n", xeGetClassName(engine.engine_class),
engine.engine_instance, engineInstance);
return &engine;
}
}
return nullptr;
}
bool IoctlHelperXe::getFabricLatency(uint32_t fabricId, uint32_t &latency, uint32_t &bandwidth) {
return false;
}
bool IoctlHelperXe::isWaitBeforeBindRequired(bool bind) const {
return true;
}
} // namespace NEO
Reference in New Issue View Git Blame Copy Permalink