/* * Copyright (C) 2023-2024 Intel Corporation * * SPDX-License-Identifier: MIT * */ #include "shared/source/os_interface/linux/xe/ioctl_helper_xe.h" #include "shared/source/command_stream/csr_definitions.h" #include "shared/source/debugger/debugger.h" #include "shared/source/execution_environment/execution_environment.h" #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/gmm_helper/gmm_helper.h" #include "shared/source/helpers/basic_math.h" #include "shared/source/helpers/bit_helpers.h" #include "shared/source/helpers/common_types.h" #include "shared/source/helpers/constants.h" #include "shared/source/helpers/engine_control.h" #include "shared/source/helpers/gfx_core_helper.h" #include "shared/source/helpers/hw_info.h" #include "shared/source/helpers/ptr_math.h" #include "shared/source/helpers/register_offsets.h" #include "shared/source/helpers/string.h" #include "shared/source/os_interface/linux/drm_buffer_object.h" #include "shared/source/os_interface/linux/drm_neo.h" #include "shared/source/os_interface/linux/engine_info.h" #include "shared/source/os_interface/linux/memory_info.h" #include "shared/source/os_interface/linux/os_context_linux.h" #include "shared/source/os_interface/linux/sys_calls.h" #include "shared/source/os_interface/os_time.h" #include "xe_drm.h" #include #include #include #include #define STRINGIFY_ME(X) return #X #define RETURN_ME(X) return X namespace NEO { const char *IoctlHelperXe::xeGetClassName(int className) { switch (className) { case DRM_XE_ENGINE_CLASS_RENDER: return "rcs"; case DRM_XE_ENGINE_CLASS_COPY: return "bcs"; case DRM_XE_ENGINE_CLASS_VIDEO_DECODE: return "vcs"; case DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE: return "vecs"; case DRM_XE_ENGINE_CLASS_COMPUTE: return "ccs"; } return "Unknown class name"; } const char *IoctlHelperXe::xeGetBindOperationName(int bindOperation) { switch (bindOperation) { case DRM_XE_VM_BIND_OP_MAP: return "MAP"; case DRM_XE_VM_BIND_OP_UNMAP: return "UNMAP"; case DRM_XE_VM_BIND_OP_MAP_USERPTR: return "MAP_USERPTR"; case DRM_XE_VM_BIND_OP_UNMAP_ALL: return "UNMAP ALL"; case DRM_XE_VM_BIND_OP_PREFETCH: return "PREFETCH"; } return "Unknown operation"; } const char *IoctlHelperXe::xeGetBindFlagsName(int bindFlags) { switch (bindFlags) { case DRM_XE_VM_BIND_FLAG_NULL: return "NULL"; } return "Unknown flag"; } const char *IoctlHelperXe::xeGetengineClassName(uint32_t engineClass) { switch (engineClass) { case DRM_XE_ENGINE_CLASS_RENDER: return "DRM_XE_ENGINE_CLASS_RENDER"; case DRM_XE_ENGINE_CLASS_COPY: return "DRM_XE_ENGINE_CLASS_COPY"; case DRM_XE_ENGINE_CLASS_VIDEO_DECODE: return "DRM_XE_ENGINE_CLASS_VIDEO_DECODE"; case DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE: return "DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE"; case DRM_XE_ENGINE_CLASS_COMPUTE: return "DRM_XE_ENGINE_CLASS_COMPUTE"; default: return "Unknown engine class"; } } IoctlHelperXe::IoctlHelperXe(Drm &drmArg) : IoctlHelper(drmArg) { xeLog("IoctlHelperXe::IoctlHelperXe\n", ""); } bool IoctlHelperXe::initialize() { xeLog("IoctlHelperXe::initialize\n", ""); drm_xe_device_query queryConfig = {}; queryConfig.query = DRM_XE_DEVICE_QUERY_CONFIG; auto retVal = IoctlHelper::ioctl(DrmIoctl::query, &queryConfig); if (retVal != 0 || queryConfig.size == 0) { return false; } auto data = std::vector(Math::divideAndRoundUp(sizeof(drm_xe_query_config) + sizeof(uint64_t) * queryConfig.size, sizeof(uint64_t)), 0); struct drm_xe_query_config *config = reinterpret_cast(data.data()); queryConfig.data = castToUint64(config); IoctlHelper::ioctl(DrmIoctl::query, &queryConfig); xeLog("DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID\t%#llx\n", config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID]); xeLog(" REV_ID\t\t\t\t%#llx\n", (config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] >> 16) & 0xff); xeLog(" DEVICE_ID\t\t\t\t%#llx\n", config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] & 0xffff); xeLog("DRM_XE_QUERY_CONFIG_FLAGS\t\t\t%#llx\n", config->info[DRM_XE_QUERY_CONFIG_FLAGS]); xeLog(" DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM\t%s\n", config->info[DRM_XE_QUERY_CONFIG_FLAGS] & DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM ? "ON" : "OFF"); xeLog("DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT\t\t%#llx\n", config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT]); xeLog("DRM_XE_QUERY_CONFIG_VA_BITS\t\t%#llx\n", config->info[DRM_XE_QUERY_CONFIG_VA_BITS]); xeLog("DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY\t\t%#llx\n", config->info[DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY]); maxExecQueuePriority = config->info[DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] & 0xffff; memset(&queryConfig, 0, sizeof(queryConfig)); queryConfig.query = DRM_XE_DEVICE_QUERY_HWCONFIG; IoctlHelper::ioctl(DrmIoctl::query, &queryConfig); auto newSize = queryConfig.size / sizeof(uint32_t); hwconfig.resize(newSize); queryConfig.data = castToUint64(hwconfig.data()); IoctlHelper::ioctl(DrmIoctl::query, &queryConfig); auto hwInfo = this->drm.getRootDeviceEnvironment().getMutableHardwareInfo(); hwInfo->platform.usDeviceID = config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] & 0xffff; hwInfo->platform.usRevId = static_cast((config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] >> 16) & 0xff); hwInfo->capabilityTable.gpuAddressSpace = (1ull << config->info[DRM_XE_QUERY_CONFIG_VA_BITS]) - 1; queryGtListData = queryData(DRM_XE_DEVICE_QUERY_GT_LIST); if (queryGtListData.empty()) { return false; } xeGtListData = reinterpret_cast(queryGtListData.data()); gtIdToTileId.resize(xeGtListData->num_gt, invalidIndex); for (auto i = 0u; i < xeGtListData->num_gt; i++) { const auto > = xeGtListData->gt_list[i]; if (gt.type == DRM_XE_QUERY_GT_TYPE_MAIN) { gtIdToTileId[gt.gt_id] = gt.tile_id; if (tileIdToGtId.size() < gt.tile_id + 1u) { tileIdToGtId.resize(gt.tile_id + 1, invalidIndex); } tileIdToGtId[gt.tile_id] = gt.gt_id; } } return true; } IoctlHelperXe::~IoctlHelperXe() { xeLog("IoctlHelperXe::~IoctlHelperXe\n", ""); } bool IoctlHelperXe::isSetPairAvailable() { return false; } bool IoctlHelperXe::isChunkingAvailable() { return false; } bool IoctlHelperXe::isVmBindAvailable() { return true; } bool IoctlHelperXe::setDomainCpu(uint32_t handle, bool writeEnable) { return false; } template std::vector IoctlHelperXe::queryData(uint32_t queryId) { struct drm_xe_device_query deviceQuery = {}; deviceQuery.query = queryId; IoctlHelper::ioctl(DrmIoctl::query, &deviceQuery); std::vector retVal(Math::divideAndRoundUp(deviceQuery.size, sizeof(DataType))); deviceQuery.data = castToUint64(retVal.data()); IoctlHelper::ioctl(DrmIoctl::query, &deviceQuery); return retVal; } template std::vector IoctlHelperXe::queryData(uint32_t queryId); template std::vector IoctlHelperXe::queryData(uint32_t queryId); std::unique_ptr IoctlHelperXe::createEngineInfo(bool isSysmanEnabled) { auto enginesData = queryData(DRM_XE_DEVICE_QUERY_ENGINES); if (enginesData.empty()) { return {}; } auto queryEngines = reinterpret_cast(enginesData.data()); auto numberHwEngines = queryEngines->num_engines; xeLog("numberHwEngines=%d\n", numberHwEngines); StackVec, 2> enginesPerTile{}; std::bitset<8> multiTileMask{}; auto hwInfo = drm.getRootDeviceEnvironment().getMutableHardwareInfo(); auto defaultEngineClass = getDefaultEngineClass(hwInfo->capabilityTable.defaultEngineType); for (auto i = 0u; i < numberHwEngines; i++) { const auto &engine = queryEngines->engines[i].instance; auto tile = engine.gt_id; multiTileMask.set(tile); EngineClassInstance engineClassInstance{}; engineClassInstance.engineClass = engine.engine_class; engineClassInstance.engineInstance = engine.engine_instance; xeLog("\t%s:%d:%d\n", xeGetClassName(engineClassInstance.engineClass), engineClassInstance.engineInstance, engine.gt_id); const bool isBaseEngineClass = engineClassInstance.engineClass == getDrmParamValue(DrmParam::engineClassCompute) || engineClassInstance.engineClass == getDrmParamValue(DrmParam::engineClassRender) || engineClassInstance.engineClass == getDrmParamValue(DrmParam::engineClassCopy); const bool isSysmanEngineClass = isSysmanEnabled && (engineClassInstance.engineClass == getDrmParamValue(DrmParam::engineClassVideo) || engineClassInstance.engineClass == getDrmParamValue(DrmParam::engineClassVideoEnhance)); if (isBaseEngineClass || isSysmanEngineClass || isExtraEngineClassAllowed(engineClassInstance.engineClass)) { if (enginesPerTile.size() <= tile) { enginesPerTile.resize(tile + 1); } enginesPerTile[tile].push_back(engineClassInstance); if (!defaultEngine && engineClassInstance.engineClass == defaultEngineClass) { defaultEngine = std::make_unique(); *defaultEngine = engine; } } } UNRECOVERABLE_IF(!defaultEngine); if (hwInfo->featureTable.flags.ftrMultiTileArch) { auto &multiTileArchInfo = hwInfo->gtSystemInfo.MultiTileArchInfo; multiTileArchInfo.IsValid = true; multiTileArchInfo.TileCount = multiTileMask.count(); multiTileArchInfo.TileMask = static_cast(multiTileMask.to_ulong()); } return std::make_unique(&drm, enginesPerTile); } inline MemoryRegion createMemoryRegionFromXeMemRegion(const drm_xe_mem_region &xeMemRegion, std::bitset<4> tilesMask) { MemoryRegion memoryRegion{}; memoryRegion.region.memoryInstance = xeMemRegion.instance; memoryRegion.region.memoryClass = xeMemRegion.mem_class; memoryRegion.probedSize = xeMemRegion.total_size; memoryRegion.unallocatedSize = xeMemRegion.total_size - xeMemRegion.used; memoryRegion.tilesMask = tilesMask; return memoryRegion; } std::unique_ptr IoctlHelperXe::createMemoryInfo() { auto memUsageData = queryData(DRM_XE_DEVICE_QUERY_MEM_REGIONS); if (memUsageData.empty()) { return {}; } constexpr auto maxSupportedTilesNumber{4u}; std::array, 64> regionTilesMask{}; for (auto i{0u}; i < xeGtListData->num_gt; i++) { const auto >Entry = xeGtListData->gt_list[i]; if (gtEntry.type != DRM_XE_QUERY_GT_TYPE_MAIN) { continue; } uint64_t nearMemRegions{gtEntry.near_mem_regions}; auto regionIndex{Math::log2(nearMemRegions)}; regionTilesMask[regionIndex].set(gtEntry.tile_id); } MemoryInfo::RegionContainer regionsContainer{}; auto xeMemRegionsData = reinterpret_cast(memUsageData.data()); for (auto i = 0u; i < xeMemRegionsData->num_mem_regions; i++) { auto &xeMemRegion{xeMemRegionsData->mem_regions[i]}; if (xeMemRegion.mem_class == DRM_XE_MEM_REGION_CLASS_SYSMEM) { // Make sure sysmem is always put at the first position regionsContainer.insert(regionsContainer.begin(), createMemoryRegionFromXeMemRegion(xeMemRegion, 0u)); } else { auto regionIndex = xeMemRegion.instance; UNRECOVERABLE_IF(regionIndex >= regionTilesMask.size()); if (auto tilesMask = regionTilesMask[regionIndex]; tilesMask.any()) { regionsContainer.push_back(createMemoryRegionFromXeMemRegion(xeMemRegion, tilesMask)); } } } if (regionsContainer.empty()) { return {}; } return std::make_unique(regionsContainer, drm); } size_t IoctlHelperXe::getLocalMemoryRegionsSize(const MemoryInfo *memoryInfo, uint32_t subDevicesCount, uint32_t tileMask) const { size_t size = 0; for (const auto &memoryRegion : memoryInfo->getLocalMemoryRegions()) { if ((memoryRegion.tilesMask & std::bitset<4>{tileMask}).any()) { size += memoryRegion.probedSize; } } return size; } void IoctlHelperXe::setupIpVersion() { auto &rootDeviceEnvironment = drm.getRootDeviceEnvironment(); auto hwInfo = rootDeviceEnvironment.getMutableHardwareInfo(); if (auto hwIpVersion = GtIpVersion{}; queryHwIpVersion(hwIpVersion)) { hwInfo->ipVersion.architecture = hwIpVersion.major; hwInfo->ipVersion.release = hwIpVersion.minor; hwInfo->ipVersion.revision = hwIpVersion.revision; } else { xeLog("No HW IP version received from drm_xe_gt. Falling back to default value."); IoctlHelper::setupIpVersion(); } } bool IoctlHelperXe::setGpuCpuTimes(TimeStampData *pGpuCpuTime, OSTime *osTime) { if (pGpuCpuTime == nullptr || osTime == nullptr) { return false; } drm_xe_device_query deviceQuery = {}; deviceQuery.query = DRM_XE_DEVICE_QUERY_ENGINE_CYCLES; auto ret = IoctlHelper::ioctl(DrmIoctl::query, &deviceQuery); if (ret != 0) { xeLog(" -> IoctlHelperXe::%s s=0x%lx r=%d\n", __FUNCTION__, deviceQuery.size, ret); return false; } std::vector retVal(deviceQuery.size); deviceQuery.data = castToUint64(retVal.data()); drm_xe_query_engine_cycles *queryEngineCycles = reinterpret_cast(retVal.data()); queryEngineCycles->clockid = CLOCK_MONOTONIC_RAW; queryEngineCycles->eci = *this->defaultEngine; ret = IoctlHelper::ioctl(DrmIoctl::query, &deviceQuery); auto nValidBits = queryEngineCycles->width; auto gpuTimestampValidBits = maxNBitValue(nValidBits); auto gpuCycles = queryEngineCycles->engine_cycles & gpuTimestampValidBits; xeLog(" -> IoctlHelperXe::%s [%d,%d] clockId=0x%x s=0x%lx nValidBits=0x%x gpuCycles=0x%x cpuTimeInNS=0x%x r=%d\n", __FUNCTION__, queryEngineCycles->eci.engine_class, queryEngineCycles->eci.engine_instance, queryEngineCycles->clockid, deviceQuery.size, nValidBits, gpuCycles, queryEngineCycles->cpu_timestamp, ret); pGpuCpuTime->gpuTimeStamp = gpuCycles; pGpuCpuTime->cpuTimeinNS = queryEngineCycles->cpu_timestamp; return ret == 0; } void IoctlHelperXe::getTopologyData(size_t nTiles, std::vector> *geomDss, std::vector> *computeDss, std::vector> *euDss, DrmQueryTopologyData &topologyData, bool &isComputeDssEmpty) { int subSliceCount = 0; int euPerDss = 0; for (auto tileId = 0u; tileId < nTiles; tileId++) { int subSliceCountPerTile = 0; for (auto byte = 0u; byte < computeDss[tileId].size(); byte++) { subSliceCountPerTile += computeDss[tileId][byte].count(); } if (subSliceCountPerTile == 0) { isComputeDssEmpty = true; for (auto byte = 0u; byte < geomDss[tileId].size(); byte++) { subSliceCountPerTile += geomDss[tileId][byte].count(); } } int euPerDssPerTile = 0; for (auto byte = 0u; byte < euDss[tileId].size(); byte++) { euPerDssPerTile += euDss[tileId][byte].count(); } // pick smallest config subSliceCount = (subSliceCount == 0) ? subSliceCountPerTile : std::min(subSliceCount, subSliceCountPerTile); euPerDss = (euPerDss == 0) ? euPerDssPerTile : std::min(euPerDss, euPerDssPerTile); // pick max config topologyData.maxSubSliceCount = std::max(topologyData.maxSubSliceCount, subSliceCountPerTile); topologyData.maxEuPerSubSlice = std::max(topologyData.maxEuPerSubSlice, euPerDssPerTile); } topologyData.sliceCount = 1; topologyData.subSliceCount = subSliceCount; topologyData.euCount = subSliceCount * euPerDss; topologyData.maxSliceCount = 1; } void IoctlHelperXe::getTopologyMap(size_t nTiles, std::vector> *dssInfo, TopologyMap &topologyMap) { for (auto tileId = 0u; tileId < nTiles; tileId++) { std::vector sliceIndices; std::vector subSliceIndices; sliceIndices.push_back(0); for (auto byte = 0u; byte < dssInfo[tileId].size(); byte++) { for (auto bit = 0u; bit < 8u; bit++) { if (dssInfo[tileId][byte].test(bit)) { auto subSliceIndex = byte * 8 + bit; subSliceIndices.push_back(subSliceIndex); } } } topologyMap[tileId].sliceIndices = std::move(sliceIndices); topologyMap[tileId].subsliceIndices = std::move(subSliceIndices); } } bool IoctlHelperXe::getTopologyDataAndMap(const HardwareInfo &hwInfo, DrmQueryTopologyData &topologyData, TopologyMap &topologyMap) { auto queryGtTopology = queryData(DRM_XE_DEVICE_QUERY_GT_TOPOLOGY); auto fillMask = [](std::vector> &vec, drm_xe_query_topology_mask *topo) { for (uint32_t j = 0; j < topo->num_bytes; j++) { vec.push_back(topo->mask[j]); } }; StackVec>, 2> geomDss; StackVec>, 2> computeDss; StackVec>, 2> euDss; auto topologySize = queryGtTopology.size(); auto dataPtr = queryGtTopology.data(); auto numTiles = tileIdToGtId.size(); geomDss.resize(numTiles); computeDss.resize(numTiles); euDss.resize(numTiles); bool receivedDssInfo = false; bool receivedEuPerDssInfo = false; while (topologySize >= sizeof(drm_xe_query_topology_mask)) { drm_xe_query_topology_mask *topo = reinterpret_cast(dataPtr); UNRECOVERABLE_IF(topo == nullptr); uint32_t gtId = topo->gt_id; auto tileId = gtIdToTileId[gtId]; if (tileId != invalidIndex) { switch (topo->type) { case DRM_XE_TOPO_DSS_GEOMETRY: fillMask(geomDss[tileId], topo); receivedDssInfo = true; break; case DRM_XE_TOPO_DSS_COMPUTE: fillMask(computeDss[tileId], topo); receivedDssInfo = true; break; case DRM_XE_TOPO_EU_PER_DSS: fillMask(euDss[tileId], topo); receivedEuPerDssInfo = true; break; default: xeLog("Unhandle GT Topo type: %d\n", topo->type); } } uint32_t itemSize = sizeof(drm_xe_query_topology_mask) + topo->num_bytes; topologySize -= itemSize; dataPtr = ptrOffset(dataPtr, itemSize); } bool isComputeDssEmpty = false; getTopologyData(numTiles, geomDss.begin(), computeDss.begin(), euDss.begin(), topologyData, isComputeDssEmpty); auto &dssInfo = isComputeDssEmpty ? geomDss : computeDss; getTopologyMap(numTiles, dssInfo.begin(), topologyMap); return receivedDssInfo && receivedEuPerDssInfo; } void IoctlHelperXe::updateBindInfo(uint32_t handle, uint64_t userPtr, uint64_t size) { std::unique_lock lock(xeLock); BindInfo b = {handle, userPtr, 0, size}; bindInfo.push_back(b); } uint16_t IoctlHelperXe::getDefaultEngineClass(const aub_stream::EngineType &defaultEngineType) { if (defaultEngineType == aub_stream::EngineType::ENGINE_CCS) { return DRM_XE_ENGINE_CLASS_COMPUTE; } else if (defaultEngineType == aub_stream::EngineType::ENGINE_RCS) { return DRM_XE_ENGINE_CLASS_RENDER; } else { /* So far defaultEngineType is either ENGINE_RCS or ENGINE_CCS */ UNRECOVERABLE_IF(true); return 0; } } /** * @brief returns caching policy for new allocation. * For system memory caching policy is write-back, otherwise it's write-combined. * * @param[in] allocationInSystemMemory flag that indicates if allocation will be allocated in system memory * * @return returns caching policy defined as DRM_XE_GEM_CPU_CACHING_WC or DRM_XE_GEM_CPU_CACHING_WB */ uint16_t IoctlHelperXe::getCpuCachingMode(bool allocationInSystemMemory) const { uint16_t cpuCachingMode = DRM_XE_GEM_CPU_CACHING_WC; if (allocationInSystemMemory) { cpuCachingMode = DRM_XE_GEM_CPU_CACHING_WB; } if (debugManager.flags.OverrideCpuCaching.get() != -1) { cpuCachingMode = debugManager.flags.OverrideCpuCaching.get(); } return cpuCachingMode; } int IoctlHelperXe::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, std::optional memPolicyMode, std::optional> memPolicyNodemask) { struct drm_xe_gem_create create = {}; uint32_t regionsSize = static_cast(memClassInstances.size()); if (!regionsSize) { xeLog("memClassInstances empty !\n", ""); return -1; } if (vmId != std::nullopt) { create.vm_id = vmId.value(); } create.size = allocSize; MemoryClassInstance mem = memClassInstances[regionsSize - 1]; std::bitset<32> memoryInstances{}; for (const auto &memoryClassInstance : memClassInstances) { memoryInstances.set(memoryClassInstance.memoryInstance); } create.placement = static_cast(memoryInstances.to_ulong()); create.cpu_caching = this->getCpuCachingMode(mem.memoryClass == drm_xe_memory_class::DRM_XE_MEM_REGION_CLASS_SYSMEM); auto ret = IoctlHelper::ioctl(DrmIoctl::gemCreate, &create); handle = create.handle; xeLog(" -> IoctlHelperXe::%s [%d,%d] vmid=0x%x s=0x%lx f=0x%x p=0x%x h=0x%x c=%hu r=%d\n", __FUNCTION__, mem.memoryClass, mem.memoryInstance, create.vm_id, create.size, create.flags, create.placement, handle, create.cpu_caching, ret); updateBindInfo(create.handle, 0u, create.size); return ret; } uint32_t IoctlHelperXe::createGem(uint64_t size, uint32_t memoryBanks) { struct drm_xe_gem_create create = {}; create.size = size; auto pHwInfo = drm.getRootDeviceEnvironment().getHardwareInfo(); auto memoryInfo = drm.getMemoryInfo(); std::bitset<32> memoryInstances{}; auto banks = std::bitset<4>(memoryBanks); size_t currentBank = 0; size_t i = 0; while (i < banks.count()) { if (banks.test(currentBank)) { auto regionClassAndInstance = memoryInfo->getMemoryRegionClassAndInstance(1u << currentBank, *pHwInfo); memoryInstances.set(regionClassAndInstance.memoryInstance); i++; } currentBank++; } if (memoryBanks == 0) { auto regionClassAndInstance = memoryInfo->getMemoryRegionClassAndInstance(memoryBanks, *pHwInfo); memoryInstances.set(regionClassAndInstance.memoryInstance); } create.placement = static_cast(memoryInstances.to_ulong()); create.cpu_caching = this->getCpuCachingMode(create.placement == drm_xe_memory_class::DRM_XE_MEM_REGION_CLASS_SYSMEM); [[maybe_unused]] auto ret = ioctl(DrmIoctl::gemCreate, &create); xeLog(" -> IoctlHelperXe::%s vmid=0x%x s=0x%lx f=0x%x p=0x%x h=0x%x c=%hu r=%d\n", __FUNCTION__, create.vm_id, create.size, create.flags, create.placement, create.handle, create.cpu_caching, ret); DEBUG_BREAK_IF(ret != 0); updateBindInfo(create.handle, 0u, create.size); return create.handle; } CacheRegion IoctlHelperXe::closAlloc() { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return CacheRegion::none; } uint16_t IoctlHelperXe::closAllocWays(CacheRegion closIndex, uint16_t cacheLevel, uint16_t numWays) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; } CacheRegion IoctlHelperXe::closFree(CacheRegion closIndex) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return CacheRegion::none; } void IoctlHelperXe::setupXeWaitUserFenceStruct(void *arg, uint32_t ctxId, uint16_t op, uint64_t addr, uint64_t value, int64_t timeout) { auto waitUserFence = reinterpret_cast(arg); waitUserFence->addr = addr; waitUserFence->op = op; waitUserFence->value = value; waitUserFence->mask = std::numeric_limits::max(); waitUserFence->timeout = timeout; waitUserFence->exec_queue_id = ctxId; } int IoctlHelperXe::xeWaitUserFence(uint32_t ctxId, uint16_t op, uint64_t addr, uint64_t value, int64_t timeout, bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) { drm_xe_wait_user_fence waitUserFence = {}; setupXeWaitUserFenceStruct(&waitUserFence, ctxId, op, addr, value, timeout); auto retVal = IoctlHelper::ioctl(DrmIoctl::gemWaitUserFence, &waitUserFence); xeLog(" -> IoctlHelperXe::%s a=0x%llx v=0x%llx T=0x%llx F=0x%x ctx=0x%x retVal=0x%x\n", __FUNCTION__, addr, value, timeout, waitUserFence.flags, ctxId, retVal); return retVal; } int IoctlHelperXe::waitUserFence(uint32_t ctxId, uint64_t address, uint64_t value, uint32_t dataWidth, int64_t timeout, uint16_t flags, bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) { xeLog(" -> IoctlHelperXe::%s a=0x%llx v=0x%llx w=0x%x T=0x%llx F=0x%x ctx=0x%x\n", __FUNCTION__, address, value, dataWidth, timeout, flags, ctxId); UNRECOVERABLE_IF(dataWidth != static_cast(Drm::ValueWidth::u64)); if (address) { return xeWaitUserFence(ctxId, DRM_XE_UFENCE_WAIT_OP_GTE, address, value, timeout, userInterrupt, externalInterruptId, allocForInterruptWait); } return 0; } uint32_t IoctlHelperXe::getAtomicAdvise(bool isNonAtomic) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; } uint32_t IoctlHelperXe::getAtomicAccess(AtomicAccessMode mode) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; } uint32_t IoctlHelperXe::getPreferredLocationAdvise() { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; } std::optional IoctlHelperXe::getPreferredLocationRegion(PreferredLocation memoryLocation, uint32_t memoryInstance) { return std::nullopt; } bool IoctlHelperXe::setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return false; } bool IoctlHelperXe::setVmBoAdviseForChunking(int32_t handle, uint64_t start, uint64_t length, uint32_t attribute, void *region) { return false; } bool IoctlHelperXe::setVmPrefetch(uint64_t start, uint64_t length, uint32_t region, uint32_t vmId) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return false; } uint32_t IoctlHelperXe::getDirectSubmissionFlag() { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; } uint16_t IoctlHelperXe::getWaitUserFenceSoftFlag() { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; }; void IoctlHelperXe::fillExecObject(ExecObject &execObject, uint32_t handle, uint64_t gpuAddress, uint32_t drmContextId, bool bindInfo, bool isMarkedForCapture) { auto execObjectXe = reinterpret_cast(execObject.data); execObjectXe->gpuAddress = gpuAddress; execObjectXe->handle = handle; } void IoctlHelperXe::logExecObject(const ExecObject &execObject, std::stringstream &logger, size_t size) { auto execObjectXe = reinterpret_cast(execObject.data); logger << "ExecBufferXe = { handle: BO-" << execObjectXe->handle << ", address range: 0x" << reinterpret_cast(execObjectXe->gpuAddress) << " }\n"; } void IoctlHelperXe::fillExecBuffer(ExecBuffer &execBuffer, uintptr_t buffersPtr, uint32_t bufferCount, uint32_t startOffset, uint32_t size, uint64_t flags, uint32_t drmContextId) { auto execBufferXe = reinterpret_cast(execBuffer.data); execBufferXe->execObject = reinterpret_cast(buffersPtr); execBufferXe->startOffset = startOffset; execBufferXe->drmContextId = drmContextId; } void IoctlHelperXe::logExecBuffer(const ExecBuffer &execBuffer, std::stringstream &logger) { auto execBufferXe = reinterpret_cast(execBuffer.data); logger << "ExecBufferXe { " << "exec object: " + std::to_string(reinterpret_cast(execBufferXe->execObject)) << ", start offset: " + std::to_string(execBufferXe->startOffset) << ", drm context id: " + std::to_string(execBufferXe->drmContextId) << " }\n"; } int IoctlHelperXe::execBuffer(ExecBuffer *execBuffer, uint64_t completionGpuAddress, TaskCountType counterValue) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); int ret = 0; if (execBuffer) { auto execBufferXe = reinterpret_cast(execBuffer->data); if (execBufferXe) { auto execObject = execBufferXe->execObject; uint32_t engine = execBufferXe->drmContextId; xeLog("EXEC ofs=%d ctx=0x%x ptr=0x%p\n", execBufferXe->startOffset, execBufferXe->drmContextId, execBufferXe->execObject); xeLog(" -> IoctlHelperXe::%s CA=0x%llx v=0x%x ctx=0x%x\n", __FUNCTION__, completionGpuAddress, counterValue, engine); struct drm_xe_sync sync[1] = {}; sync[0].type = DRM_XE_SYNC_TYPE_USER_FENCE; sync[0].flags = DRM_XE_SYNC_FLAG_SIGNAL; sync[0].addr = completionGpuAddress; sync[0].timeline_value = counterValue; struct drm_xe_exec exec = {}; exec.exec_queue_id = engine; exec.num_syncs = 1; exec.syncs = reinterpret_cast(&sync); exec.address = execObject->gpuAddress + execBufferXe->startOffset; exec.num_batch_buffer = 1; ret = IoctlHelper::ioctl(DrmIoctl::gemExecbuffer2, &exec); xeLog("r=0x%x batch=0x%lx\n", ret, exec.address); if (debugManager.flags.PrintCompletionFenceUsage.get()) { std::cout << "Completion fence submitted." << " GPU address: " << std::hex << completionGpuAddress << std::dec << ", value: " << counterValue << std::endl; } } } return ret; } bool IoctlHelperXe::completionFenceExtensionSupported(const bool isVmBindAvailable) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return isVmBindAvailable; } uint64_t IoctlHelperXe::getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool bindLock, bool readOnlyResource) { uint64_t ret = 0; xeLog(" -> IoctlHelperXe::%s %d %d %d %d %d\n", __FUNCTION__, bindCapture, bindImmediate, bindMakeResident, bindLock, readOnlyResource); if (bindCapture) { ret |= DRM_XE_VM_BIND_FLAG_DUMPABLE; } return ret; } int IoctlHelperXe::queryDistances(std::vector &queryItems, std::vector &distanceInfos) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; } std::optional IoctlHelperXe::getHasPageFaultParamId() { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return {}; }; uint32_t IoctlHelperXe::getEuStallFdParameter() { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0u; } std::unique_ptr IoctlHelperXe::createVmControlExtRegion(const std::optional ®ionInstanceClass) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return {}; } uint32_t IoctlHelperXe::getFlagsForVmCreate(bool disableScratch, bool enablePageFault, bool useVmBind) { xeLog(" -> IoctlHelperXe::%s %d,%d,%d\n", __FUNCTION__, disableScratch, enablePageFault, useVmBind); uint32_t flags = DRM_XE_VM_CREATE_FLAG_LR_MODE; if (enablePageFault) { flags |= DRM_XE_VM_CREATE_FLAG_FAULT_MODE; } return flags; } uint32_t IoctlHelperXe::createContextWithAccessCounters(GemContextCreateExt &gcc) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; } uint32_t IoctlHelperXe::createCooperativeContext(GemContextCreateExt &gcc) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; } void IoctlHelperXe::fillVmBindExtSetPat(VmBindExtSetPatT &vmBindExtSetPat, uint64_t patIndex, uint64_t nextExtension) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); } void IoctlHelperXe::fillVmBindExtUserFence(VmBindExtUserFenceT &vmBindExtUserFence, uint64_t fenceAddress, uint64_t fenceValue, uint64_t nextExtension) { xeLog(" -> IoctlHelperXe::%s 0x%lx 0x%lx\n", __FUNCTION__, fenceAddress, fenceValue); auto xeBindExtUserFence = reinterpret_cast(vmBindExtUserFence); UNRECOVERABLE_IF(!xeBindExtUserFence); xeBindExtUserFence->tag = UserFenceExtension::tagValue; xeBindExtUserFence->addr = fenceAddress; xeBindExtUserFence->value = fenceValue; } void IoctlHelperXe::setVmBindUserFence(VmBindParams &vmBind, VmBindExtUserFenceT vmBindUserFence) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); vmBind.userFence = castToUint64(vmBindUserFence); return; } std::optional IoctlHelperXe::getCopyClassSaturatePCIECapability() { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return {}; } std::optional IoctlHelperXe::getCopyClassSaturateLinkCapability() { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return {}; } uint32_t IoctlHelperXe::getVmAdviseAtomicAttribute() { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; } int IoctlHelperXe::vmBind(const VmBindParams &vmBindParams) { return xeVmBind(vmBindParams, true); } int IoctlHelperXe::vmUnbind(const VmBindParams &vmBindParams) { return xeVmBind(vmBindParams, false); } int IoctlHelperXe::getResetStats(ResetStats &resetStats, uint32_t *status, ResetStatsFault *resetStatsFault) { return ioctl(DrmIoctl::getResetStats, &resetStats); } UuidRegisterResult IoctlHelperXe::registerUuid(const std::string &uuid, uint32_t uuidClass, uint64_t ptr, uint64_t size) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return {}; } UuidRegisterResult IoctlHelperXe::registerStringClassUuid(const std::string &uuid, uint64_t ptr, uint64_t size) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return {}; } int IoctlHelperXe::unregisterUuid(uint32_t handle) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; } bool IoctlHelperXe::isContextDebugSupported() { return false; } int IoctlHelperXe::setContextDebugFlag(uint32_t drmContextId) { return 0; } bool IoctlHelperXe::isDebugAttachAvailable() { return true; } int IoctlHelperXe::getDrmParamValue(DrmParam drmParam) const { xeLog(" -> IoctlHelperXe::%s 0x%x %s\n", __FUNCTION__, drmParam, getDrmParamString(drmParam).c_str()); switch (drmParam) { case DrmParam::memoryClassDevice: return DRM_XE_MEM_REGION_CLASS_VRAM; case DrmParam::memoryClassSystem: return DRM_XE_MEM_REGION_CLASS_SYSMEM; case DrmParam::engineClassRender: return DRM_XE_ENGINE_CLASS_RENDER; case DrmParam::engineClassCopy: return DRM_XE_ENGINE_CLASS_COPY; case DrmParam::engineClassVideo: return DRM_XE_ENGINE_CLASS_VIDEO_DECODE; case DrmParam::engineClassVideoEnhance: return DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE; case DrmParam::engineClassCompute: return DRM_XE_ENGINE_CLASS_COMPUTE; case DrmParam::engineClassInvalid: return -1; case DrmParam::execDefault: return DRM_XE_ENGINE_CLASS_COMPUTE; case DrmParam::execBlt: return DRM_XE_ENGINE_CLASS_COPY; case DrmParam::execRender: return DRM_XE_ENGINE_CLASS_RENDER; default: return getDrmParamValueBase(drmParam); } } int IoctlHelperXe::getDrmParamValueBase(DrmParam drmParam) const { return static_cast(drmParam); } int IoctlHelperXe::ioctl(DrmIoctl request, void *arg) { int ret = -1; xeLog(" => IoctlHelperXe::%s 0x%x\n", __FUNCTION__, request); switch (request) { case DrmIoctl::getparam: { auto getParam = reinterpret_cast(arg); ret = 0; switch (getParam->param) { case static_cast(DrmParam::paramCsTimestampFrequency): { *getParam->value = xeGtListData->gt_list[defaultEngine->gt_id].reference_clock; } break; default: ret = -1; } xeLog(" -> IoctlHelperXe::ioctl Getparam 0x%x/0x%x r=%d\n", getParam->param, *getParam->value, ret); } break; case DrmIoctl::query: { Query *query = static_cast(arg); QueryItem *queryItems = reinterpret_cast(query->itemsPtr); for (auto i = 0u; i < query->numItems; i++) { auto &queryItem = queryItems[i]; if (queryItem.queryId != static_cast(DrmParam::queryHwconfigTable)) { xeLog("error: bad query 0x%x\n", queryItem.queryId); return -1; } auto queryDataSize = static_cast(hwconfig.size() * sizeof(uint32_t)); if (queryItem.length == 0) { queryItem.length = queryDataSize; } else { UNRECOVERABLE_IF(queryItem.length != queryDataSize); memcpy_s(reinterpret_cast(queryItem.dataPtr), queryItem.length, hwconfig.data(), queryItem.length); } xeLog(" -> IoctlHelperXe::ioctl Query id=0x%x f=0x%x len=%d\n", static_cast(queryItem.queryId), static_cast(queryItem.flags), queryItem.length); ret = 0; } } break; case DrmIoctl::gemUserptr: { GemUserPtr *d = static_cast(arg); updateBindInfo(0, d->userPtr, d->userSize); ret = 0; xeLog(" -> IoctlHelperXe::ioctl GemUserptr p=0x%llx s=0x%llx f=0x%x h=0x%x r=%d\n", d->userPtr, d->userSize, d->flags, d->handle, ret); xeShowBindTable(); } break; case DrmIoctl::gemContextDestroy: { GemContextDestroy *d = static_cast(arg); struct drm_xe_exec_queue_destroy destroy = {}; destroy.exec_queue_id = d->contextId; ret = IoctlHelper::ioctl(request, &destroy); xeLog(" -> IoctlHelperXe::ioctl GemContextDestroryExt ctx=0x%x r=%d\n", d->contextId, ret); } break; case DrmIoctl::gemContextGetparam: { GemContextParam *d = static_cast(arg); auto addressSpace = drm.getRootDeviceEnvironment().getHardwareInfo()->capabilityTable.gpuAddressSpace; ret = 0; switch (d->param) { case static_cast(DrmParam::contextParamGttSize): d->value = addressSpace + 1u; break; default: ret = -1; break; } xeLog(" -> IoctlHelperXe::ioctl GemContextGetparam r=%d\n", ret); } break; case DrmIoctl::gemContextSetparam: { GemContextParam *gemContextParam = static_cast(arg); switch (gemContextParam->param) { case static_cast(DrmParam::contextParamEngines): { auto contextEngine = reinterpret_cast *>(gemContextParam->value); if (!contextEngine || contextEngine->numEnginesInContext == 0) { break; } auto numEngines = contextEngine->numEnginesInContext; contextParamEngine.resize(numEngines); memcpy_s(contextParamEngine.data(), numEngines * sizeof(uint64_t), contextEngine->enginesData, numEngines * sizeof(uint64_t)); ret = 0; } break; default: ret = -1; break; } xeLog(" -> IoctlHelperXe::ioctl GemContextSetparam r=%d\n", ret); } break; case DrmIoctl::gemClose: { struct GemClose *d = static_cast(arg); int found = -1; xeShowBindTable(); bool isUserptr = false; for (unsigned int i = 0; i < bindInfo.size(); i++) { if (d->handle && d->handle == bindInfo[i].handle) { found = i; break; } if (d->userptr && d->userptr == bindInfo[i].userptr) { found = i; isUserptr = true; break; } } if (found != -1) { xeLog(" removing %d: 0x%x 0x%lx 0x%lx\n", found, bindInfo[found].handle, bindInfo[found].userptr, bindInfo[found].addr); { std::unique_lock lock(xeLock); bindInfo.erase(bindInfo.begin() + found); } if (isUserptr) { // nothing to do under XE ret = 0; } else { ret = IoctlHelper::ioctl(request, arg); } } else { ret = 0; // let it pass trough for now } xeLog(" -> IoctlHelperXe::ioctl GemClose found=%d h=0x%x r=%d\n", found, d->handle, ret); } break; case DrmIoctl::gemVmCreate: { GemVmControl *vmControl = static_cast(arg); struct drm_xe_vm_create args = {}; args.flags = vmControl->flags; if (drm.getRootDeviceEnvironment().executionEnvironment.getDebuggingMode() != DebuggingMode::disabled) { args.extensions = reinterpret_cast(allocateDebugMetadata()); } ret = IoctlHelper::ioctl(request, &args); if (drm.getRootDeviceEnvironment().executionEnvironment.getDebuggingMode() != DebuggingMode::disabled) { args.extensions = reinterpret_cast(freeDebugMetadata(reinterpret_cast(args.extensions))); } vmControl->vmId = args.vm_id; xeLog(" -> IoctlHelperXe::ioctl gemVmCreate f=0x%x vmid=0x%x r=%d\n", vmControl->flags, vmControl->vmId, ret); } break; case DrmIoctl::gemVmDestroy: { GemVmControl *d = static_cast(arg); struct drm_xe_vm_destroy args = {}; args.vm_id = d->vmId; ret = IoctlHelper::ioctl(request, &args); xeLog(" -> IoctlHelperXe::ioctl GemVmDestroy vmid=0x%x r=%d\n", d->vmId, ret); } break; case DrmIoctl::gemMmapOffset: { GemMmapOffset *d = static_cast(arg); struct drm_xe_gem_mmap_offset mmo = {}; mmo.handle = d->handle; ret = IoctlHelper::ioctl(request, &mmo); d->offset = mmo.offset; xeLog(" -> IoctlHelperXe::ioctl GemMmapOffset h=0x%x o=0x%x f=0x%x r=%d\n", d->handle, d->offset, d->flags, ret); } break; case DrmIoctl::getResetStats: { ResetStats *resetStats = static_cast(arg); drm_xe_exec_queue_get_property getProperty{}; getProperty.exec_queue_id = resetStats->contextId; getProperty.property = DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN; ret = IoctlHelper::ioctl(request, &getProperty); resetStats->batchPending = static_cast(getProperty.value); xeLog(" -> IoctlHelperXe::ioctl GetResetStats ctx=0x%x r=%d value=%llu\n", resetStats->contextId, ret, getProperty.value); } break; case DrmIoctl::primeFdToHandle: { PrimeHandle *prime = static_cast(arg); ret = IoctlHelper::ioctl(request, arg); xeLog(" ->PrimeFdToHandle h=0x%x f=0x%x d=0x%x r=%d\n", prime->handle, prime->flags, prime->fileDescriptor, ret); } break; case DrmIoctl::primeHandleToFd: { PrimeHandle *prime = static_cast(arg); ret = IoctlHelper::ioctl(request, arg); xeLog(" ->PrimeHandleToFd h=0x%x f=0x%x d=0x%x r=%d\n", prime->handle, prime->flags, prime->fileDescriptor, ret); } break; case DrmIoctl::gemCreate: { drm_xe_gem_create *gemCreate = static_cast(arg); ret = IoctlHelper::ioctl(request, arg); xeLog(" -> IoctlHelperXe::ioctl GemCreate h=0x%x s=0x%lx p=0x%x f=0x%x vmid=0x%x r=%d\n", gemCreate->handle, gemCreate->size, gemCreate->placement, gemCreate->flags, gemCreate->vm_id, ret); } break; case DrmIoctl::debuggerOpen: { ret = debuggerOpenIoctl(request, arg); } break; case DrmIoctl::metadataCreate: { ret = debuggerMetadataCreateIoctl(request, arg); } break; case DrmIoctl::metadataDestroy: { ret = debuggerMetadataDestroyIoctl(request, arg); } break; default: xeLog("Not handled 0x%x\n", request); UNRECOVERABLE_IF(true); } return ret; } void IoctlHelperXe::xeShowBindTable() { if (debugManager.flags.PrintXeLogs.get()) { std::unique_lock lock(xeLock); xeLog("show bind: ( )\n", ""); for (unsigned int i = 0; i < bindInfo.size(); i++) { xeLog(" %3d x%08x x%016lx x%016lx x%016lx\n", i, bindInfo[i].handle, bindInfo[i].userptr, bindInfo[i].addr, bindInfo[i].size); } } } int IoctlHelperXe::createDrmContext(Drm &drm, OsContextLinux &osContext, uint32_t drmVmId, uint32_t deviceIndex) { drm_xe_exec_queue_create create = {}; uint32_t drmContextId = 0; xeLog("createDrmContext VM=0x%x\n", drmVmId); drm.bindDrmContext(drmContextId, deviceIndex, osContext.getEngineType(), osContext.isEngineInstanced()); size_t n = contextParamEngine.size(); UNRECOVERABLE_IF(n == 0); create.vm_id = drmVmId; create.width = 1; create.instances = castToUint64(contextParamEngine.data()); create.num_placements = contextParamEngine.size(); std::array extProperties{}; uint32_t extPropertyIndex = 0; auto &gfxCoreHelper = drm.getRootDeviceEnvironment().getHelper(); if ((contextParamEngine[0].engine_class == DRM_XE_ENGINE_CLASS_RENDER) || (contextParamEngine[0].engine_class == DRM_XE_ENGINE_CLASS_COMPUTE)) { if (gfxCoreHelper.isRunaloneModeRequired(drm.getRootDeviceEnvironment().executionEnvironment.getDebuggingMode())) { extProperties[extPropertyIndex].base.next_extension = 0; extProperties[extPropertyIndex].base.name = DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY; extProperties[extPropertyIndex].property = getRunaloneExtProperty(); extProperties[extPropertyIndex].value = 1; extPropertyIndex++; } } setContextProperties(osContext, &extProperties, extPropertyIndex); if (extPropertyIndex > 0) { create.extensions = castToUint64(&extProperties[0]); } int ret = IoctlHelper::ioctl(DrmIoctl::gemContextCreateExt, &create); drmContextId = create.exec_queue_id; xeLog("%s:%d (%d) vmid=0x%x ctx=0x%x r=0x%x\n", xeGetClassName(contextParamEngine[0].engine_class), contextParamEngine[0].engine_instance, create.num_placements, drmVmId, drmContextId, ret); if (ret != 0) { UNRECOVERABLE_IF(true); } return drmContextId; } int IoctlHelperXe::xeVmBind(const VmBindParams &vmBindParams, bool isBind) { constexpr int invalidIndex = -1; auto gmmHelper = drm.getRootDeviceEnvironment().getGmmHelper(); int ret = -1; const char *operation = isBind ? "bind" : "unbind"; int index = invalidIndex; if (isBind) { for (auto i = 0u; i < bindInfo.size(); i++) { if (vmBindParams.handle && vmBindParams.handle == bindInfo[i].handle) { index = i; break; } if (vmBindParams.userptr && vmBindParams.userptr == bindInfo[i].userptr) { index = i; break; } } } else // unbind { auto address = gmmHelper->decanonize(vmBindParams.start); for (auto i = 0u; i < bindInfo.size(); i++) { if (address == bindInfo[i].addr) { index = i; break; } } } if (index != invalidIndex) { drm_xe_sync sync[1] = {}; sync[0].type = DRM_XE_SYNC_TYPE_USER_FENCE; sync[0].flags = DRM_XE_SYNC_FLAG_SIGNAL; auto xeBindExtUserFence = reinterpret_cast(vmBindParams.userFence); UNRECOVERABLE_IF(!xeBindExtUserFence); UNRECOVERABLE_IF(xeBindExtUserFence->tag != UserFenceExtension::tagValue); sync[0].addr = xeBindExtUserFence->addr; sync[0].timeline_value = xeBindExtUserFence->value; drm_xe_vm_bind bind = {}; bind.vm_id = vmBindParams.vmId; bind.num_binds = 1; bind.num_syncs = 1; bind.syncs = reinterpret_cast(&sync); bind.bind.range = vmBindParams.length; bind.bind.addr = gmmHelper->decanonize(vmBindParams.start); bind.bind.obj_offset = vmBindParams.offset; bind.bind.pat_index = static_cast(vmBindParams.patIndex); bind.bind.extensions = vmBindParams.extensions; bind.bind.flags = static_cast(vmBindParams.flags); if (isBind) { bind.bind.op = DRM_XE_VM_BIND_OP_MAP; bind.bind.obj = vmBindParams.handle; if (bindInfo[index].userptr) { bind.bind.op = DRM_XE_VM_BIND_OP_MAP_USERPTR; bind.bind.obj = 0; bind.bind.obj_offset = bindInfo[index].userptr; } } else { bind.bind.op = DRM_XE_VM_BIND_OP_UNMAP; bind.bind.obj = 0; if (bindInfo[index].userptr) { bind.bind.obj_offset = bindInfo[index].userptr; } } bindInfo[index].addr = bind.bind.addr; ret = IoctlHelper::ioctl(DrmIoctl::gemVmBind, &bind); xeLog(" vm=%d obj=0x%x off=0x%llx range=0x%llx addr=0x%llx operation=%d(%s) flags=%d(%s) nsy=%d pat=%hu ret=%d\n", bind.vm_id, bind.bind.obj, bind.bind.obj_offset, bind.bind.range, bind.bind.addr, bind.bind.op, xeGetBindOperationName(bind.bind.op), bind.bind.flags, xeGetBindFlagsName(bind.bind.flags), bind.num_syncs, bind.bind.pat_index, ret); if (ret != 0) { xeLog("error: %s\n", operation); return ret; } constexpr auto oneSecTimeout = 1000000000; constexpr auto infiniteTimeout = -1; bool debuggingEnabled = drm.getRootDeviceEnvironment().executionEnvironment.isDebuggingEnabled(); auto timeout = debuggingEnabled ? infiniteTimeout : oneSecTimeout; return xeWaitUserFence(bind.exec_queue_id, DRM_XE_UFENCE_WAIT_OP_EQ, sync[0].addr, sync[0].timeline_value, timeout, false, NEO::InterruptId::notUsed, nullptr); } xeLog("error: -> IoctlHelperXe::%s %s index=%d vmid=0x%x h=0x%x s=0x%llx o=0x%llx l=0x%llx f=0x%llx pat=%hu r=%d\n", __FUNCTION__, operation, index, vmBindParams.vmId, vmBindParams.handle, vmBindParams.start, vmBindParams.offset, vmBindParams.length, vmBindParams.flags, vmBindParams.patIndex, ret); return ret; } std::string IoctlHelperXe::getDrmParamString(DrmParam drmParam) const { switch (drmParam) { case DrmParam::contextCreateExtSetparam: return "ContextCreateExtSetparam"; case DrmParam::contextCreateFlagsUseExtensions: return "ContextCreateFlagsUseExtensions"; case DrmParam::contextEnginesExtLoadBalance: return "ContextEnginesExtLoadBalance"; case DrmParam::contextParamEngines: return "ContextParamEngines"; case DrmParam::contextParamGttSize: return "ContextParamGttSize"; case DrmParam::contextParamPersistence: return "ContextParamPersistence"; case DrmParam::contextParamPriority: return "ContextParamPriority"; case DrmParam::contextParamRecoverable: return "ContextParamRecoverable"; case DrmParam::contextParamSseu: return "ContextParamSseu"; case DrmParam::contextParamVm: return "ContextParamVm"; case DrmParam::engineClassRender: return "EngineClassRender"; case DrmParam::engineClassCompute: return "EngineClassCompute"; case DrmParam::engineClassCopy: return "EngineClassCopy"; case DrmParam::engineClassVideo: return "EngineClassVideo"; case DrmParam::engineClassVideoEnhance: return "EngineClassVideoEnhance"; case DrmParam::engineClassInvalid: return "EngineClassInvalid"; case DrmParam::engineClassInvalidNone: return "EngineClassInvalidNone"; case DrmParam::execBlt: return "ExecBlt"; case DrmParam::execDefault: return "ExecDefault"; case DrmParam::execNoReloc: return "ExecNoReloc"; case DrmParam::execRender: return "ExecRender"; case DrmParam::memoryClassDevice: return "MemoryClassDevice"; case DrmParam::memoryClassSystem: return "MemoryClassSystem"; case DrmParam::mmapOffsetWb: return "MmapOffsetWb"; case DrmParam::mmapOffsetWc: return "MmapOffsetWc"; case DrmParam::paramHasPooledEu: return "ParamHasPooledEu"; case DrmParam::paramEuTotal: return "ParamEuTotal"; case DrmParam::paramSubsliceTotal: return "ParamSubsliceTotal"; case DrmParam::paramMinEuInPool: return "ParamMinEuInPool"; case DrmParam::paramCsTimestampFrequency: return "ParamCsTimestampFrequency"; case DrmParam::paramHasVmBind: return "ParamHasVmBind"; case DrmParam::paramHasPageFault: return "ParamHasPageFault"; case DrmParam::queryEngineInfo: return "QueryEngineInfo"; case DrmParam::queryHwconfigTable: return "QueryHwconfigTable"; case DrmParam::queryComputeSlices: return "QueryComputeSlices"; case DrmParam::queryMemoryRegions: return "QueryMemoryRegions"; case DrmParam::queryTopologyInfo: return "QueryTopologyInfo"; case DrmParam::tilingNone: return "TilingNone"; case DrmParam::tilingY: return "TilingY"; default: return "DrmParam::"; } } inline std::string getDirectoryWithFrequencyFiles(int tileId, int gtId) { return "/device/tile" + std::to_string(tileId) + "/gt" + std::to_string(gtId) + "/freq0"; } std::string IoctlHelperXe::getFileForMaxGpuFrequency() const { return getFileForMaxGpuFrequencyOfSubDevice(0 /* tileId */); } std::string IoctlHelperXe::getFileForMaxGpuFrequencyOfSubDevice(int tileId) const { return getDirectoryWithFrequencyFiles(tileId, tileIdToGtId[tileId]) + "/max_freq"; } std::string IoctlHelperXe::getFileForMaxMemoryFrequencyOfSubDevice(int tileId) const { return getDirectoryWithFrequencyFiles(tileId, tileIdToGtId[tileId]) + "/rp0_freq"; } bool IoctlHelperXe::getFabricLatency(uint32_t fabricId, uint32_t &latency, uint32_t &bandwidth) { return false; } bool IoctlHelperXe::isWaitBeforeBindRequired(bool bind) const { return true; } bool IoctlHelperXe::setGemTiling(void *setTiling) { return true; } bool IoctlHelperXe::getGemTiling(void *setTiling) { return true; } void IoctlHelperXe::fillBindInfoForIpcHandle(uint32_t handle, size_t size) { xeLog(" -> IoctlHelperXe::%s s=0x%lx h=0x%x\n", __FUNCTION__, size, handle); updateBindInfo(handle, 0, size); } bool IoctlHelperXe::isImmediateVmBindRequired() const { return true; } void IoctlHelperXe::insertEngineToContextParams(ContextParamEngines<> &contextParamEngines, uint32_t engineId, const EngineClassInstance *engineClassInstance, uint32_t tileId, bool hasVirtualEngines) { auto engines = reinterpret_cast(contextParamEngines.enginesData); if (engineClassInstance) { engines[engineId].engine_class = engineClassInstance->engineClass; engines[engineId].engine_instance = engineClassInstance->engineInstance; engines[engineId].gt_id = tileIdToGtId[tileId]; contextParamEngines.numEnginesInContext = std::max(contextParamEngines.numEnginesInContext, engineId + 1); } } void IoctlHelperXe::registerBOBindHandle(Drm *drm, DrmAllocation *drmAllocation) { DrmResourceClass resourceClass = DrmResourceClass::maxSize; switch (drmAllocation->getAllocationType()) { case AllocationType::debugContextSaveArea: resourceClass = DrmResourceClass::contextSaveArea; break; case AllocationType::debugSbaTrackingBuffer: resourceClass = DrmResourceClass::sbaTrackingBuffer; break; case AllocationType::debugModuleArea: resourceClass = DrmResourceClass::moduleHeapDebugArea; break; default: return; break; } uint64_t gpuAddress = drmAllocation->getGpuAddress(); auto handle = drm->registerResource(resourceClass, &gpuAddress, sizeof(gpuAddress)); drmAllocation->addRegisteredBoBindHandle(handle); auto &bos = drmAllocation->getBOs(); for (auto bo : bos) { if (!bo) { continue; } bo->addBindExtHandle(handle); bo->markForCapture(); bo->requireImmediateBinding(true); } } bool IoctlHelperXe::getFdFromVmExport(uint32_t vmId, uint32_t flags, int32_t *fd) { return false; } void IoctlHelperXe::setContextProperties(const OsContextLinux &osContext, void *extProperties, uint32_t &extIndexInOut) { auto &ext = *reinterpret_cast *>(extProperties); if (osContext.isLowPriority()) { ext[extIndexInOut].base.name = DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY; ext[extIndexInOut].property = DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY; ext[extIndexInOut].value = 0; if (extIndexInOut > 0) { ext[extIndexInOut - 1].base.next_extension = castToUint64(&ext[extIndexInOut]); } extIndexInOut++; } } bool IoctlHelperXe::perfOpenEuStallStream(uint32_t euStallFdParameter, std::array &properties, int32_t *stream) { return false; } bool IoctlHelperXe::perfDisableEuStallStream(int32_t *stream) { return false; } bool IoctlHelperXe::getEuStallProperties(std::array &properties, uint64_t dssBufferSize, uint64_t samplingRate, uint64_t pollPeriod, uint64_t engineInstance, uint64_t notifyNReports) { return false; } unsigned int IoctlHelperXe::getIoctlRequestValue(DrmIoctl ioctlRequest) const { xeLog(" -> IoctlHelperXe::%s 0x%x\n", __FUNCTION__, ioctlRequest); switch (ioctlRequest) { case DrmIoctl::gemClose: RETURN_ME(DRM_IOCTL_GEM_CLOSE); case DrmIoctl::gemVmCreate: RETURN_ME(DRM_IOCTL_XE_VM_CREATE); case DrmIoctl::gemVmDestroy: RETURN_ME(DRM_IOCTL_XE_VM_DESTROY); case DrmIoctl::gemMmapOffset: RETURN_ME(DRM_IOCTL_XE_GEM_MMAP_OFFSET); case DrmIoctl::gemCreate: RETURN_ME(DRM_IOCTL_XE_GEM_CREATE); case DrmIoctl::gemExecbuffer2: RETURN_ME(DRM_IOCTL_XE_EXEC); case DrmIoctl::gemVmBind: RETURN_ME(DRM_IOCTL_XE_VM_BIND); case DrmIoctl::query: RETURN_ME(DRM_IOCTL_XE_DEVICE_QUERY); case DrmIoctl::gemContextCreateExt: RETURN_ME(DRM_IOCTL_XE_EXEC_QUEUE_CREATE); case DrmIoctl::gemContextDestroy: RETURN_ME(DRM_IOCTL_XE_EXEC_QUEUE_DESTROY); case DrmIoctl::gemWaitUserFence: RETURN_ME(DRM_IOCTL_XE_WAIT_USER_FENCE); case DrmIoctl::primeFdToHandle: RETURN_ME(DRM_IOCTL_PRIME_FD_TO_HANDLE); case DrmIoctl::primeHandleToFd: RETURN_ME(DRM_IOCTL_PRIME_HANDLE_TO_FD); case DrmIoctl::getResetStats: RETURN_ME(DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY); case DrmIoctl::debuggerOpen: case DrmIoctl::metadataCreate: case DrmIoctl::metadataDestroy: return getIoctlRequestValueDebugger(ioctlRequest); default: UNRECOVERABLE_IF(true); return 0; } } std::string IoctlHelperXe::getIoctlString(DrmIoctl ioctlRequest) const { switch (ioctlRequest) { case DrmIoctl::gemClose: STRINGIFY_ME(DRM_IOCTL_GEM_CLOSE); case DrmIoctl::gemVmCreate: STRINGIFY_ME(DRM_IOCTL_XE_VM_CREATE); case DrmIoctl::gemVmDestroy: STRINGIFY_ME(DRM_IOCTL_XE_VM_DESTROY); case DrmIoctl::gemMmapOffset: STRINGIFY_ME(DRM_IOCTL_XE_GEM_MMAP_OFFSET); case DrmIoctl::gemCreate: STRINGIFY_ME(DRM_IOCTL_XE_GEM_CREATE); case DrmIoctl::gemExecbuffer2: STRINGIFY_ME(DRM_IOCTL_XE_EXEC); case DrmIoctl::gemVmBind: STRINGIFY_ME(DRM_IOCTL_XE_VM_BIND); case DrmIoctl::query: STRINGIFY_ME(DRM_IOCTL_XE_DEVICE_QUERY); case DrmIoctl::gemContextCreateExt: STRINGIFY_ME(DRM_IOCTL_XE_EXEC_QUEUE_CREATE); case DrmIoctl::gemContextDestroy: STRINGIFY_ME(DRM_IOCTL_XE_EXEC_QUEUE_DESTROY); case DrmIoctl::gemWaitUserFence: STRINGIFY_ME(DRM_IOCTL_XE_WAIT_USER_FENCE); case DrmIoctl::primeFdToHandle: STRINGIFY_ME(DRM_IOCTL_PRIME_FD_TO_HANDLE); case DrmIoctl::primeHandleToFd: STRINGIFY_ME(DRM_IOCTL_PRIME_HANDLE_TO_FD); case DrmIoctl::debuggerOpen: STRINGIFY_ME(DRM_IOCTL_XE_EUDEBUG_CONNECT); case DrmIoctl::metadataCreate: STRINGIFY_ME(DRM_IOCTL_XE_DEBUG_METADATA_CREATE); case DrmIoctl::metadataDestroy: STRINGIFY_ME(DRM_IOCTL_XE_DEBUG_METADATA_DESTROY); case DrmIoctl::getResetStats: STRINGIFY_ME(DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY); default: return "???"; } } } // namespace NEO