diff --git a/CMakeLists.txt b/CMakeLists.txt index 475bf981b7..bc52c5a60f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright (C) 2018-2023 Intel Corporation +# Copyright (C) 2018-2024 Intel Corporation # # SPDX-License-Identifier: MIT # @@ -430,6 +430,14 @@ if(NOT DISABLE_LIBVA) endif() endif() +if(UNIX) + if(NOT DEFINED NUMA_HEADERS_DIR) + get_filename_component(NUMA_HEADERS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/numa" ABSOLUTE) + message(STATUS "NUMA config headers dir: ${NUMA_HEADERS_DIR}") + include_directories("${NUMA_HEADERS_DIR}") + endif() +endif() + add_subdirectory_unique(third_party/gtest) message(STATUS "AUB_STREAM_DIR = ${AUB_STREAM_DIR}") diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index 78830f1314..a6eb13fe85 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -492,6 +492,8 @@ DECLARE_DEBUG_VARIABLE(int32_t, CompactL3FlushEventPacket, -1, "Compact COMPUTE_ DECLARE_DEBUG_VARIABLE(int32_t, UseDynamicEventPacketsCount, -1, "Use dynamic estimation for event packet count based on a given device configuration, -1: default , 0: disabled, 1: enabled") DECLARE_DEBUG_VARIABLE(int32_t, SignalAllEventPackets, -1, "All packets of event are signaled, reset and waited/synchronized, -1: default, 0: disabled, 1: enabled") DECLARE_DEBUG_VARIABLE(int32_t, EnableBcsSwControlWa, -1, "Enable BCS WA via BCSSWCONTROL MMIO. -1: default, 0: disabled, 1: if src in system mem, 2: if dst in system mem, 3: if src and dst in system mem, 4: always") +DECLARE_DEBUG_VARIABLE(bool, EnableHostAllocationMemPolicy, false, "Enables Memory Policy for host allocation") +DECLARE_DEBUG_VARIABLE(int32_t, OverrideHostAllocationMemPolicyMode, -1, "Override Memory Policy mode for host allocation -1: default (use the system configuration), 0: MPOL_DEFAULT, 1: MPOL_PREFERRED, 2: MPOL_BIND, 3: MPOL_INTERLEAVED, 4: MPOL_LOCAL, 5: MPOL_PREFERRED_MANY") /* IMPLICIT SCALING */ DECLARE_DEBUG_VARIABLE(int32_t, EnableWalkerPartition, -1, "-1: default, 0: disable, 1: enable, Enables Walker Partitioning via WPARID.") diff --git a/shared/source/os_interface/linux/CMakeLists.txt b/shared/source/os_interface/linux/CMakeLists.txt index 5693c2da5f..d75ccb5737 100644 --- a/shared/source/os_interface/linux/CMakeLists.txt +++ b/shared/source/os_interface/linux/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright (C) 2019-2023 Intel Corporation +# Copyright (C) 2019-2024 Intel Corporation # # SPDX-License-Identifier: MIT # @@ -62,6 +62,8 @@ set(NEO_CORE_OS_INTERFACE_LINUX ${CMAKE_CURRENT_SOURCE_DIR}/engine_info.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory_info.h ${CMAKE_CURRENT_SOURCE_DIR}/memory_info.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/numa_library.h + ${CMAKE_CURRENT_SOURCE_DIR}/numa_library.cpp ${CMAKE_CURRENT_SOURCE_DIR}/os_context_linux.cpp ${CMAKE_CURRENT_SOURCE_DIR}/os_context_linux.h ${CMAKE_CURRENT_SOURCE_DIR}/os_inc.h diff --git a/shared/source/os_interface/linux/drm_allocation.h b/shared/source/os_interface/linux/drm_allocation.h index 40fd533cba..74f78c2d17 100644 --- a/shared/source/os_interface/linux/drm_allocation.h +++ b/shared/source/os_interface/linux/drm_allocation.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -118,6 +118,9 @@ class DrmAllocation : public GraphicsAllocation { size_t getMmapSize() { return this->mmapSize; } void setMmapSize(size_t size) { this->mmapSize = size; } + void setUsmHostAllocation(bool flag) { usmHostAllocation = flag; } + bool isUsmHostAllocation() { return usmHostAllocation; } + OsContextLinux *getOsContext() const { return this->osContext; } @@ -149,5 +152,7 @@ class DrmAllocation : public GraphicsAllocation { void *mmapPtr = nullptr; size_t mmapSize = 0u; + + bool usmHostAllocation = false; }; } // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 87c45b7e2b..4ce681f645 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -451,7 +451,7 @@ GraphicsAllocation *DrmMemoryManager::allocateUSMHostGraphicsMemory(const Alloca bo->peekAddress(), cSize, MemoryPool::system4KBPages); - + allocation->setUsmHostAllocation(true); allocation->setReservedAddressRange(reinterpret_cast(gpuAddress), cSize); bo.release(); @@ -1732,6 +1732,7 @@ inline std::unique_ptr DrmMemoryManager::makeDrmAllocation(const allocation->storageInfo = allocationData.storageInfo; allocation->setFlushL3Required(allocationData.flags.flushL3); allocation->setUncacheable(allocationData.flags.uncacheable); + allocation->setUsmHostAllocation(allocationData.flags.isUSMHostAllocation); return allocation; } @@ -1868,7 +1869,7 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryInDevicePool(const A } BufferObject *DrmMemoryManager::createBufferObjectInMemoryRegion(uint32_t rootDeviceIndex, Gmm *gmm, AllocationType allocationType, uint64_t gpuAddress, - size_t size, uint32_t memoryBanks, size_t maxOsContextCount, int32_t pairHandle, bool isSystemMemoryPool) { + size_t size, uint32_t memoryBanks, size_t maxOsContextCount, int32_t pairHandle, bool isSystemMemoryPool, bool isUsmHostAllocation) { auto drm = &getDrm(rootDeviceIndex); auto memoryInfo = drm->getMemoryInfo(); if (!memoryInfo) { @@ -1882,9 +1883,9 @@ BufferObject *DrmMemoryManager::createBufferObjectInMemoryRegion(uint32_t rootDe auto banks = std::bitset<4>(memoryBanks); if (banks.count() > 1) { - ret = memoryInfo->createGemExtWithMultipleRegions(memoryBanks, size, handle, patIndex); + ret = memoryInfo->createGemExtWithMultipleRegions(memoryBanks, size, handle, patIndex, isUsmHostAllocation, reinterpret_cast(gpuAddress)); } else { - ret = memoryInfo->createGemExtWithSingleRegion(memoryBanks, size, handle, patIndex, pairHandle); + ret = memoryInfo->createGemExtWithSingleRegion(memoryBanks, size, handle, patIndex, pairHandle, isUsmHostAllocation, reinterpret_cast(gpuAddress)); } if (ret != 0) { @@ -1902,6 +1903,7 @@ BufferObject *DrmMemoryManager::createBufferObjectInMemoryRegion(uint32_t rootDe } size_t DrmMemoryManager::getSizeOfChunk(size_t allocSize) { + size_t chunkSize = MemoryConstants::chunkThreshold; size_t chunkMask = (~(MemoryConstants::chunkThreshold - 1)); size_t numChunk = debugManager.flags.NumberOfBOChunks.get(); @@ -1946,7 +1948,7 @@ bool DrmMemoryManager::createDrmChunkedAllocation(Drm *drm, DrmAllocation *alloc auto gmm = allocation->getGmm(0u); auto patIndex = drm->getPatIndex(gmm, allocation->getAllocationType(), CacheRegion::defaultRegion, CachePolicy::writeBack, false, !allocation->isAllocatedInLocalMemoryPool()); - int ret = memoryInfo->createGemExtWithMultipleRegions(memoryBanks, boSize, handle, patIndex, -1, true, numOfChunks); + int ret = memoryInfo->createGemExtWithMultipleRegions(memoryBanks, boSize, handle, patIndex, -1, true, numOfChunks, allocation->isUsmHostAllocation(), reinterpret_cast(boAddress)); if (ret != 0) { return false; } @@ -2019,7 +2021,7 @@ bool DrmMemoryManager::createDrmAllocation(Drm *drm, DrmAllocation *allocation, auto gmm = allocation->getGmm(handleId); auto boSize = alignUp(gmm->gmmResourceInfo->getSizeAllocation(), MemoryConstants::pageSize64k); bos[handleId] = createBufferObjectInMemoryRegion(allocation->getRootDeviceIndex(), gmm, allocation->getAllocationType(), boAddress, boSize, memoryBanks, maxOsContextCount, pairHandle, - !allocation->isAllocatedInLocalMemoryPool()); + !allocation->isAllocatedInLocalMemoryPool(), allocation->isUsmHostAllocation()); if (nullptr == bos[handleId]) { return false; } @@ -2159,7 +2161,7 @@ DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData & auto pointerDiff = ptrDiff(cpuPointer, cpuBasePointer); std::unique_ptr bo(this->createBufferObjectInMemoryRegion(allocationData.rootDeviceIndex, nullptr, allocationData.type, reinterpret_cast(cpuPointer), alignedSize, 0u, maxOsContextCount, -1, - MemoryPoolHelper::isSystemMemoryPool(memoryPool))); + MemoryPoolHelper::isSystemMemoryPool(memoryPool), allocationData.flags.isUSMHostAllocation)); if (!bo) { releaseGpuRange(reinterpret_cast(preferredAddress), totalSizeToAlloc, allocationData.rootDeviceIndex); @@ -2350,7 +2352,7 @@ GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const auto patIndex = drm.getPatIndex(nullptr, allocationData.type, CacheRegion::defaultRegion, CachePolicy::writeBack, false, MemoryPoolHelper::isSystemMemoryPool(memoryPool)); - int ret = memoryInfo->createGemExt(memRegions, currentSize, handle, patIndex, {}, -1, useChunking, numOfChunks); + int ret = memoryInfo->createGemExt(memRegions, currentSize, handle, patIndex, {}, -1, useChunking, numOfChunks, allocationData.flags.isUSMHostAllocation, reinterpret_cast(preferredAddress)); if (ret) { this->munmapFunction(cpuPointer, totalSizeToAlloc); diff --git a/shared/source/os_interface/linux/drm_memory_manager.h b/shared/source/os_interface/linux/drm_memory_manager.h index 6b1c932a04..529c4c7ba6 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.h +++ b/shared/source/os_interface/linux/drm_memory_manager.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -73,7 +73,7 @@ class DrmMemoryManager : public MemoryManager { size_t selectAlignmentAndHeap(size_t size, HeapIndex *heap) override; void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override; MOCKABLE_VIRTUAL BufferObject *createBufferObjectInMemoryRegion(uint32_t rootDeviceIndex, Gmm *gmm, AllocationType allocationType, uint64_t gpuAddress, size_t size, - uint32_t memoryBanks, size_t maxOsContextCount, int32_t pairHandle, bool isSystemMemoryPool); + uint32_t memoryBanks, size_t maxOsContextCount, int32_t pairHandle, bool isSystemMemoryPool, bool isUsmHostAllocation); bool hasPageFaultsEnabled(const Device &neoDevice) override; bool isKmdMigrationAvailable(uint32_t rootDeviceIndex) override; diff --git a/shared/source/os_interface/linux/i915_prelim.h b/shared/source/os_interface/linux/i915_prelim.h index 07391b548a..ddf40a589a 100644 --- a/shared/source/os_interface/linux/i915_prelim.h +++ b/shared/source/os_interface/linux/i915_prelim.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022-2023 Intel Corporation + * Copyright (C) 2022-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -64,6 +64,7 @@ using NEO::PrelimI915::prelim_drm_i915_gem_clos_free; using NEO::PrelimI915::prelim_drm_i915_gem_clos_reserve; using NEO::PrelimI915::prelim_drm_i915_gem_context_param_acc; using NEO::PrelimI915::prelim_drm_i915_gem_create_ext; +using NEO::PrelimI915::prelim_drm_i915_gem_create_ext_memory_policy; using NEO::PrelimI915::prelim_drm_i915_gem_create_ext_setparam; using NEO::PrelimI915::prelim_drm_i915_gem_create_ext_vm_private; using NEO::PrelimI915::prelim_drm_i915_gem_engine_class; diff --git a/shared/source/os_interface/linux/ioctl_helper.h b/shared/source/os_interface/linux/ioctl_helper.h index e1e45d0e27..40d269f089 100644 --- a/shared/source/os_interface/linux/ioctl_helper.h +++ b/shared/source/os_interface/linux/ioctl_helper.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Intel Corporation + * Copyright (C) 2021-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -84,7 +84,7 @@ class IoctlHelper { virtual bool isSetPairAvailable() = 0; virtual bool isChunkingAvailable() = 0; virtual bool isVmBindAvailable() = 0; - virtual int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) = 0; + virtual int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, std::optional memPolicyMode, std::optional> memPolicyNodemask) = 0; virtual uint32_t createGem(uint64_t size, uint32_t memoryBanks) = 0; virtual CacheRegion closAlloc() = 0; virtual uint16_t closAllocWays(CacheRegion closIndex, uint16_t cacheLevel, uint16_t numWays) = 0; @@ -209,7 +209,7 @@ class IoctlHelperUpstream : public IoctlHelperI915 { bool isSetPairAvailable() override; bool isChunkingAvailable() override; bool isVmBindAvailable() override; - int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) override; + int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, std::optional memPolicyMode, std::optional> memPolicyNodemask) override; CacheRegion closAlloc() override; uint16_t closAllocWays(CacheRegion closIndex, uint16_t cacheLevel, uint16_t numWays) override; CacheRegion closFree(CacheRegion closIndex) override; @@ -269,7 +269,7 @@ class IoctlHelperImpl : public IoctlHelperUpstream { return std::make_unique>(drm); } - int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) override; + int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, std::optional memPolicyMode, std::optional> memPolicyNodemask) override; std::vector translateToMemoryRegions(const std::vector ®ionInfo) override; unsigned int getIoctlRequestValue(DrmIoctl ioctlRequest) const override; std::string getIoctlString(DrmIoctl ioctlRequest) const override; @@ -283,7 +283,7 @@ class IoctlHelperPrelim20 : public IoctlHelperI915 { bool isSetPairAvailable() override; bool isChunkingAvailable() override; bool isVmBindAvailable() override; - int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) override; + int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, std::optional memPolicyMode, std::optional> memPolicyNodemask) override; CacheRegion closAlloc() override; uint16_t closAllocWays(CacheRegion closIndex, uint16_t cacheLevel, uint16_t numWays) override; CacheRegion closFree(CacheRegion closIndex) override; diff --git a/shared/source/os_interface/linux/ioctl_helper_prelim.cpp b/shared/source/os_interface/linux/ioctl_helper_prelim.cpp index 9fff8533da..aaf8756cbb 100644 --- a/shared/source/os_interface/linux/ioctl_helper_prelim.cpp +++ b/shared/source/os_interface/linux/ioctl_helper_prelim.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Intel Corporation + * Copyright (C) 2021-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -147,7 +147,7 @@ bool IoctlHelperPrelim20::isVmBindAvailable() { return vmBindSupported; } -int IoctlHelperPrelim20::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) { +int IoctlHelperPrelim20::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, std::optional memPolicyMode, std::optional> memPolicyNodemask) { uint32_t regionsSize = static_cast(memClassInstances.size()); std::vector regions(regionsSize); for (uint32_t i = 0; i < regionsSize; i++) { @@ -166,12 +166,22 @@ int IoctlHelperPrelim20::createGemExt(const MemRegionsVec &memClassInstances, si prelim_drm_i915_gem_create_ext_vm_private vmPrivate{}; prelim_drm_i915_gem_create_ext_setparam pairSetparamRegion{}; prelim_drm_i915_gem_create_ext_setparam chunkingParamRegion{}; + prelim_drm_i915_gem_create_ext_memory_policy memPolicy{}; if (vmId != std::nullopt) { vmPrivate.base.name = PRELIM_I915_GEM_CREATE_EXT_VM_PRIVATE; vmPrivate.vm_id = vmId.value(); } + if (memPolicyMode != std::nullopt) { + UNRECOVERABLE_IF(memPolicyNodemask == std::nullopt); + memPolicy.base.name = PRELIM_I915_GEM_CREATE_EXT_MEMORY_POLICY; + memPolicy.mode = memPolicyMode.value(); + memPolicy.flags = 0; + memPolicy.nodemask_max = static_cast(memPolicyNodemask.value().size()); + memPolicy.nodemask_ptr = reinterpret_cast(memPolicyNodemask.value().data()); + } + if (pairHandle != -1) { pairSetparamRegion.base.name = PRELIM_I915_GEM_CREATE_EXT_SETPARAM; pairSetparamRegion.param.param = PRELIM_I915_OBJECT_PARAM | PRELIM_I915_PARAM_SET_PAIR; @@ -186,17 +196,27 @@ int IoctlHelperPrelim20::createGemExt(const MemRegionsVec &memClassInstances, si UNRECOVERABLE_IF(chunkingSize & (MemoryConstants::pageSize64k - 1)); chunkingParamRegion.param.data = chunkingSize; setparamRegion.base.next_extension = reinterpret_cast(&chunkingParamRegion); + if (memPolicyMode != std::nullopt) { + chunkingParamRegion.base.next_extension = reinterpret_cast(&memPolicy); + } } else { + auto *lastExtension = &(setparamRegion.base); if (vmId != std::nullopt) { - vmPrivate.base.next_extension = reinterpret_cast(&pairSetparamRegion); setparamRegion.base.next_extension = reinterpret_cast(&vmPrivate); - } else if (pairHandle != -1) { - setparamRegion.base.next_extension = reinterpret_cast(&pairSetparamRegion); + lastExtension = &(vmPrivate.base); + } + if (pairHandle != -1) { + lastExtension->next_extension = reinterpret_cast(&pairSetparamRegion); + lastExtension = &(pairSetparamRegion.base); + } + if (memPolicyMode != std::nullopt) { + lastExtension->next_extension = reinterpret_cast(&memPolicy); } } prelim_drm_i915_gem_create_ext createExt{}; createExt.size = allocSize; + createExt.extensions = reinterpret_cast(&setparamRegion); printDebugString(debugManager.flags.PrintBOCreateDestroyResult.get(), stdout, "Performing GEM_CREATE_EXT with { size: %lu, param: 0x%llX", @@ -208,10 +228,17 @@ int IoctlHelperPrelim20::createGemExt(const MemRegionsVec &memClassInstances, si printDebugString(debugManager.flags.PrintBOCreateDestroyResult.get(), stdout, ", memory class: %d, memory instance: %d", region.memory_class, region.memory_instance); } + if (memPolicyMode != std::nullopt) { + printDebugString(debugManager.flags.PrintBOCreateDestroyResult.get(), stdout, + ", memory policy:{ mode: %d, nodemask_max: 0x%d, nodemask_ptr: 0x%llX }", + memPolicy.mode, + memPolicy.nodemask_max, + memPolicy.nodemask_ptr); + } printDebugString(debugManager.flags.PrintBOCreateDestroyResult.get(), stdout, "%s", " }\n"); } - auto ret = IoctlHelper::ioctl(DrmIoctl::gemCreateExt, &createExt); + auto ret = ioctl(DrmIoctl::gemCreateExt, &createExt); if (isChunked) { printDebugString(debugManager.flags.PrintBOChunkingLogs.get(), stdout, @@ -221,7 +248,6 @@ int IoctlHelperPrelim20::createGemExt(const MemRegionsVec &memClassInstances, si chunkingParamRegion.param.data, numOfChunks); } - printDebugString(debugManager.flags.PrintBOCreateDestroyResult.get(), stdout, "GEM_CREATE_EXT has returned: %d BO-%u with size: %lu\n", ret, createExt.handle, createExt.size); handle = createExt.handle; return ret; diff --git a/shared/source/os_interface/linux/ioctl_helper_upstream.cpp b/shared/source/os_interface/linux/ioctl_helper_upstream.cpp index a0f8dcda45..48d1c6f7b8 100644 --- a/shared/source/os_interface/linux/ioctl_helper_upstream.cpp +++ b/shared/source/os_interface/linux/ioctl_helper_upstream.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Intel Corporation + * Copyright (C) 2021-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -33,7 +33,7 @@ bool IoctlHelperUpstream::isVmBindAvailable() { return false; } -int IoctlHelperUpstream::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) { +int IoctlHelperUpstream::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, std::optional memPolicyMode, std::optional> memPolicyNodemask) { bool isPatIndexValid = (patIndex != CommonConstants::unsupportedPatIndex) && (patIndex <= std::numeric_limits::max()); bool useSetPat = this->isSetPatSupported && isPatIndexValid; diff --git a/shared/source/os_interface/linux/local/dg1/ioctl_helper_dg1.cpp b/shared/source/os_interface/linux/local/dg1/ioctl_helper_dg1.cpp index 3122a753e7..31e4d0281d 100644 --- a/shared/source/os_interface/linux/local/dg1/ioctl_helper_dg1.cpp +++ b/shared/source/os_interface/linux/local/dg1/ioctl_helper_dg1.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Intel Corporation + * Copyright (C) 2021-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -21,8 +21,8 @@ constexpr static auto gfxProduct = IGFX_DG1; extern bool isQueryDrmTip(const std::vector &queryInfo); template <> -int IoctlHelperImpl::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) { - auto ret = IoctlHelperUpstream::createGemExt(memClassInstances, allocSize, handle, patIndex, vmId, pairHandle, isChunked, numOfChunks); +int IoctlHelperImpl::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, std::optional memPolicyMode, std::optional> memPolicyNodemask) { + auto ret = IoctlHelperUpstream::createGemExt(memClassInstances, allocSize, handle, patIndex, vmId, pairHandle, isChunked, numOfChunks, memPolicyMode, memPolicyNodemask); if (ret == 0) { return ret; } diff --git a/shared/source/os_interface/linux/memory_info.cpp b/shared/source/os_interface/linux/memory_info.cpp index 7718326dff..d393c3d571 100644 --- a/shared/source/os_interface/linux/memory_info.cpp +++ b/shared/source/os_interface/linux/memory_info.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Intel Corporation + * Copyright (C) 2021-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -14,6 +14,7 @@ #include "shared/source/helpers/gfx_core_helper.h" #include "shared/source/helpers/hw_info.h" #include "shared/source/os_interface/linux/drm_neo.h" +#include "shared/source/os_interface/linux/numa_library.h" #include @@ -29,6 +30,9 @@ MemoryInfo::MemoryInfo(const RegionContainer ®ionInfo, const Drm &inputDrm) [&](const MemoryRegion &memoryRegionInfo) { return (memoryRegionInfo.region.memoryClass == memoryClassDevice); }); + + memPolicySupported = Linux::NumaLibrary::init() && debugManager.flags.EnableHostAllocationMemPolicy.get(); + memPolicyMode = debugManager.flags.OverrideHostAllocationMemPolicyMode.get(); } void MemoryInfo::assignRegionsFromDistances(const std::vector &distances) { @@ -56,8 +60,19 @@ void MemoryInfo::assignRegionsFromDistances(const std::vector &dis } } -int MemoryInfo::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) { - return this->drm.getIoctlHelper()->createGemExt(memClassInstances, allocSize, handle, patIndex, vmId, pairHandle, isChunked, numOfChunks); +int MemoryInfo::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, bool isUSMHostAllocation, void *addr) { + std::vector memPolicyNodeMask; + int mode = -1; + if (memPolicySupported && + isUSMHostAllocation && + Linux::NumaLibrary::getMemPolicy(&mode, memPolicyNodeMask, addr)) { + if (memPolicyMode != -1) { + mode = memPolicyMode; + } + return this->drm.getIoctlHelper()->createGemExt(memClassInstances, allocSize, handle, patIndex, vmId, pairHandle, isChunked, numOfChunks, mode, memPolicyNodeMask); + } else { + return this->drm.getIoctlHelper()->createGemExt(memClassInstances, allocSize, handle, patIndex, vmId, pairHandle, isChunked, numOfChunks, std::nullopt, std::nullopt); + } } uint32_t MemoryInfo::getTileIndex(uint32_t memoryBank) { @@ -109,7 +124,7 @@ void MemoryInfo::printRegionSizes() { } } -int MemoryInfo::createGemExtWithSingleRegion(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle) { +int MemoryInfo::createGemExtWithSingleRegion(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle, bool isUSMHostAllocation, void *addr) { auto pHwInfo = this->drm.getRootDeviceEnvironment().getHardwareInfo(); auto regionClassAndInstance = getMemoryRegionClassAndInstance(memoryBanks, *pHwInfo); MemRegionsVec region = {regionClassAndInstance}; @@ -121,11 +136,11 @@ int MemoryInfo::createGemExtWithSingleRegion(uint32_t memoryBanks, size_t allocS } } uint32_t numOfChunks = 0; - auto ret = createGemExt(region, allocSize, handle, patIndex, vmId, pairHandle, false, numOfChunks); + auto ret = createGemExt(region, allocSize, handle, patIndex, vmId, pairHandle, false, numOfChunks, isUSMHostAllocation, addr); return ret; } -int MemoryInfo::createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex) { +int MemoryInfo::createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, bool isUSMHostAllocation, void *addr) { auto pHwInfo = this->drm.getRootDeviceEnvironment().getHardwareInfo(); auto banks = std::bitset<4>(memoryBanks); MemRegionsVec memRegions{}; @@ -140,11 +155,11 @@ int MemoryInfo::createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t all currentBank++; } uint32_t numOfChunks = 0; - auto ret = createGemExt(memRegions, allocSize, handle, patIndex, {}, -1, false, numOfChunks); + auto ret = createGemExt(memRegions, allocSize, handle, patIndex, {}, -1, false, numOfChunks, isUSMHostAllocation, addr); return ret; } -int MemoryInfo::createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) { +int MemoryInfo::createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, bool isUSMHostAllocation, void *addr) { auto pHwInfo = this->drm.getRootDeviceEnvironment().getHardwareInfo(); auto banks = std::bitset<4>(memoryBanks); MemRegionsVec memRegions{}; @@ -158,7 +173,7 @@ int MemoryInfo::createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t all } currentBank++; } - auto ret = createGemExt(memRegions, allocSize, handle, patIndex, {}, pairHandle, isChunked, numOfChunks); + auto ret = createGemExt(memRegions, allocSize, handle, patIndex, {}, pairHandle, isChunked, numOfChunks, isUSMHostAllocation, addr); return ret; } diff --git a/shared/source/os_interface/linux/memory_info.h b/shared/source/os_interface/linux/memory_info.h index 62a0d67d92..4b278aa2c0 100644 --- a/shared/source/os_interface/linux/memory_info.h +++ b/shared/source/os_interface/linux/memory_info.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2023 Intel Corporation + * Copyright (C) 2019-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -26,7 +26,7 @@ class MemoryInfo { void assignRegionsFromDistances(const std::vector &distances); - MOCKABLE_VIRTUAL int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks); + MOCKABLE_VIRTUAL int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, bool isUSMHostAllocation, void *addr); MemoryClassInstance getMemoryRegionClassAndInstance(uint32_t memoryBank, const HardwareInfo &hwInfo); @@ -38,18 +38,20 @@ class MemoryInfo { uint32_t getTileIndex(uint32_t memoryBank); - MOCKABLE_VIRTUAL int createGemExtWithSingleRegion(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle); - MOCKABLE_VIRTUAL int createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex); - MOCKABLE_VIRTUAL int createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle, bool isChunked, uint32_t numOfChunks); + MOCKABLE_VIRTUAL int createGemExtWithSingleRegion(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle, bool isUSMHostAllocation, void *addr); + MOCKABLE_VIRTUAL int createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, bool isUSMHostAllocation, void *addr); + MOCKABLE_VIRTUAL int createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, bool isUSMHostAllocation, void *addr); const RegionContainer &getDrmRegionInfos() const { return drmQueryRegions; } + bool isMemPolicySupported() const { return memPolicySupported; } protected: const Drm &drm; const RegionContainer drmQueryRegions; const MemoryRegion &systemMemoryRegion; - + bool memPolicySupported; + int memPolicyMode; RegionContainer localMemoryRegions; }; diff --git a/shared/source/os_interface/linux/numa_library.cpp b/shared/source/os_interface/linux/numa_library.cpp new file mode 100644 index 0000000000..b4fcca324a --- /dev/null +++ b/shared/source/os_interface/linux/numa_library.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/linux/numa_library.h" + +#include "shared/source/helpers/debug_helpers.h" + +#include +#include +#include + +namespace NEO { +namespace Linux { + +std::unique_ptr NumaLibrary::osLibrary(nullptr); +NumaLibrary::OsLibraryLoadPtr NumaLibrary::osLibraryLoadFunction(NEO::OsLibrary::load); +NumaLibrary::GetMemPolicyPtr NumaLibrary::getMemPolicyFunction(nullptr); +NumaLibrary::NumaAvailablePtr NumaLibrary::numaAvailableFunction(nullptr); +NumaLibrary::NumaMaxNodePtr NumaLibrary::numaMaxNodeFunction(nullptr); +int NumaLibrary::maxNode(-1); +bool NumaLibrary::numaLoaded(false); + +bool NumaLibrary::init() { + osLibrary.reset(osLibraryLoadFunction(std::string(numaLibNameStr))); + numaLoaded = false; + numaAvailableFunction = nullptr; + numaMaxNodeFunction = nullptr; + getMemPolicyFunction = nullptr; + if (osLibrary) { + DEBUG_BREAK_IF(!osLibrary->isLoaded()); + numaAvailableFunction = reinterpret_cast(osLibrary->getProcAddress(std::string(procNumaAvailableStr))); + numaMaxNodeFunction = reinterpret_cast(osLibrary->getProcAddress(std::string(procNumaMaxNodeStr))); + getMemPolicyFunction = reinterpret_cast(osLibrary->getProcAddress(std::string(procGetMemPolicyStr))); + if (numaAvailableFunction && numaMaxNodeFunction && getMemPolicyFunction) { + if ((*numaAvailableFunction)() == 0) { + maxNode = (*numaMaxNodeFunction)(); + numaLoaded = maxNode > 0; + } + } + } + return numaLoaded; +} + +bool NumaLibrary::getMemPolicy(int *mode, std::vector &nodeMask, void *addr) { + if (numaLoaded) { + // re-initialize vector with size maxNode; + std::vector(maxNode + 1, 0).swap(nodeMask); + return (*getMemPolicyFunction)(mode, nodeMask.data(), maxNode + 1, addr, MPOL_F_MEMS_ALLOWED) != -1; + } + return false; +} + +} // namespace Linux +} // namespace NEO diff --git a/shared/source/os_interface/linux/numa_library.h b/shared/source/os_interface/linux/numa_library.h new file mode 100644 index 0000000000..55b10d97c7 --- /dev/null +++ b/shared/source/os_interface/linux/numa_library.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include "shared/source/os_interface/linux/os_library_linux.h" +#include "shared/source/os_interface/os_library.h" + +#include +#include +#include +#include +namespace NEO { +namespace Linux { + +class NumaLibrary { + public: + static bool init(); + static bool isLoaded() { return numaLoaded; } + static bool getMemPolicy(int *mode, std::vector &nodeMask, void *addr); + + protected: + static constexpr const char *numaLibNameStr = "libnuma.so"; + static constexpr const char *procGetMemPolicyStr = "get_mempolicy"; + static constexpr const char *procNumaAvailableStr = "numa_available"; + static constexpr const char *procNumaMaxNodeStr = "numa_max_node"; + + using OsLibraryLoadPtr = std::add_pointer::type; + using GetMemPolicyPtr = std::add_pointer::type; + using NumaAvailablePtr = std::add_pointer::type; + using NumaMaxNodePtr = std::add_pointer::type; + + static std::unique_ptr osLibrary; + static OsLibraryLoadPtr osLibraryLoadFunction; + static GetMemPolicyPtr getMemPolicyFunction; + static NumaAvailablePtr numaAvailableFunction; + static NumaMaxNodePtr numaMaxNodeFunction; + static int maxNode; + static bool numaLoaded; +}; +} // namespace Linux +} // namespace NEO diff --git a/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp b/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp index 70058f00b5..7d0702dee9 100644 --- a/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp +++ b/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -498,7 +498,7 @@ uint16_t IoctlHelperXe::getCpuCachingMode() { return cpuCachingMode; } -int IoctlHelperXe::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) { +int IoctlHelperXe::createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, std::optional memPolicyMode, std::optional> memPolicyNodemask) { struct drm_xe_gem_create create = {}; uint32_t regionsSize = static_cast(memClassInstances.size()); diff --git a/shared/source/os_interface/linux/xe/ioctl_helper_xe.h b/shared/source/os_interface/linux/xe/ioctl_helper_xe.h index 571d926d5d..eddfdd41a5 100644 --- a/shared/source/os_interface/linux/xe/ioctl_helper_xe.h +++ b/shared/source/os_interface/linux/xe/ioctl_helper_xe.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -50,7 +50,7 @@ class IoctlHelperXe : public IoctlHelper { bool isSetPairAvailable() override; bool isChunkingAvailable() override; bool isVmBindAvailable() override; - int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) override; + int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, std::optional memPolicyMode, std::optional> memPolicyNodemask) override; uint32_t createGem(uint64_t size, uint32_t memoryBanks) override; CacheRegion closAlloc() override; uint16_t closAllocWays(CacheRegion closIndex, uint16_t cacheLevel, uint16_t numWays) override; diff --git a/shared/test/common/libult/linux/drm_mock_prelim_context.cpp b/shared/test/common/libult/linux/drm_mock_prelim_context.cpp index 8055602106..1ad21a32fe 100644 --- a/shared/test/common/libult/linux/drm_mock_prelim_context.cpp +++ b/shared/test/common/libult/linux/drm_mock_prelim_context.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022-2023 Intel Corporation + * Copyright (C) 2022-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -164,21 +164,38 @@ int DrmMockPrelimContext::handlePrelimRequest(DrmIoctl request, void *arg) { return EINVAL; } + prelim_drm_i915_gem_create_ext_setparam *pairSetparamRegion = nullptr; + prelim_drm_i915_gem_create_ext_setparam *chunkingSetparamRegion = nullptr; prelim_drm_i915_gem_create_ext_vm_private *vmPrivateExt = nullptr; - if (extension->base.next_extension != 0) { - - prelim_drm_i915_gem_create_ext_setparam *pairSetparamRegion = nullptr; - pairSetparamRegion = reinterpret_cast(extension->base.next_extension); - if (pairSetparamRegion->base.name == PRELIM_I915_GEM_CREATE_EXT_SETPARAM) { - if ((pairSetparamRegion->base.name & PRELIM_I915_PARAM_SET_PAIR) == 0) { - return EINVAL; - } - } else { - vmPrivateExt = reinterpret_cast(extension->base.next_extension); - if (vmPrivateExt->base.name != PRELIM_I915_GEM_CREATE_EXT_VM_PRIVATE) { + prelim_drm_i915_gem_create_ext_memory_policy *memPolicyExt = nullptr; + void *next_extension = reinterpret_cast(extension->base.next_extension); + while (next_extension != 0) { + auto *setparamCandidate = reinterpret_cast(next_extension); + if (setparamCandidate->base.name == PRELIM_I915_GEM_CREATE_EXT_SETPARAM) { + if ((setparamCandidate->param.param & PRELIM_I915_PARAM_SET_PAIR) != 0) { + pairSetparamRegion = setparamCandidate; + } else if ((setparamCandidate->param.param & PRELIM_I915_PARAM_SET_CHUNK_SIZE) != 0) { + chunkingSetparamRegion = setparamCandidate; + } else { return EINVAL; } + next_extension = reinterpret_cast(setparamCandidate->base.next_extension); + continue; } + auto *vmPrivateCandidate = reinterpret_cast(next_extension); + if (vmPrivateCandidate->base.name == PRELIM_I915_GEM_CREATE_EXT_VM_PRIVATE) { + vmPrivateExt = vmPrivateCandidate; + next_extension = reinterpret_cast(vmPrivateCandidate->base.next_extension); + continue; + } + auto *memPolicyCandidate = reinterpret_cast(next_extension); + if (memPolicyCandidate->base.name == PRELIM_I915_GEM_CREATE_EXT_MEMORY_POLICY) { + memPolicyExt = memPolicyCandidate; + next_extension = reinterpret_cast(memPolicyCandidate->base.next_extension); + continue; + } + // incorrect extension detected + return EINVAL; } auto data = reinterpret_cast(extension->param.data); @@ -194,6 +211,20 @@ int DrmMockPrelimContext::handlePrelimRequest(DrmIoctl request, void *arg) { receivedCreateGemExt->vmPrivateExt = CreateGemExt::VmPrivate{vmPrivateExt->vm_id}; } + if (memPolicyExt != nullptr) { + receivedCreateGemExt->memPolicyExt = CreateGemExt::MemPolicy{memPolicyExt->mode, std::vector()}; + auto *memPolicyPtr = reinterpret_cast(memPolicyExt->nodemask_ptr); + for (auto i = 0u; i < memPolicyExt->nodemask_max; i++) { + receivedCreateGemExt->memPolicyExt.nodeMask.value().push_back(memPolicyPtr[i]); + } + } + if (pairSetparamRegion != nullptr) { + receivedCreateGemExt->pairSetParamExt = CreateGemExt::SetParam{pairSetparamRegion->param.handle, pairSetparamRegion->param.size, pairSetparamRegion->param.param}; + } + if (chunkingSetparamRegion != nullptr) { + receivedCreateGemExt->chunkingSetParamExt = CreateGemExt::SetParam{chunkingSetparamRegion->param.handle, chunkingSetparamRegion->param.size, chunkingSetparamRegion->param.param}; + } + receivedCreateGemExt->memoryRegions.clear(); for (uint32_t i = 0; i < extension->param.size; i++) { receivedCreateGemExt->memoryRegions.push_back({data[i].memoryClass, data[i].memoryInstance}); diff --git a/shared/test/common/libult/linux/drm_mock_prelim_context.h b/shared/test/common/libult/linux/drm_mock_prelim_context.h index c59ceab91d..75ea2fea6d 100644 --- a/shared/test/common/libult/linux/drm_mock_prelim_context.h +++ b/shared/test/common/libult/linux/drm_mock_prelim_context.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022-2023 Intel Corporation + * Copyright (C) 2022-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -37,6 +37,8 @@ struct CreateGemExt { uint64_t param{0}; }; std::optional setParamExt{}; + std::optional pairSetParamExt{}; + std::optional chunkingSetParamExt{}; struct MemoryClassInstance { uint16_t memoryClass{0}; @@ -48,6 +50,12 @@ struct CreateGemExt { std::optional vmId{}; }; VmPrivate vmPrivateExt{}; + + struct MemPolicy { + std::optional mode{}; + std::optional> nodeMask{}; + }; + MemPolicy memPolicyExt{}; }; struct GemContextParamAcc { diff --git a/shared/test/common/mocks/mock_os_library.h b/shared/test/common/mocks/mock_os_library.h index c91a4c7ec1..7729532a08 100644 --- a/shared/test/common/mocks/mock_os_library.h +++ b/shared/test/common/mocks/mock_os_library.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Intel Corporation + * Copyright (C) 2021-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -9,6 +9,7 @@ #include "shared/source/os_interface/os_library.h" +#include class MockOsLibrary : public NEO::OsLibrary { public: MockOsLibrary(void *procAddress, bool isLoaded) : getProcAddressReturn{procAddress}, isLoadedReturn{isLoaded} {} @@ -40,3 +41,16 @@ class MockOsLibrary : public NEO::OsLibrary { return ptr; } }; + +class MockOsLibraryCustom : public MockOsLibrary { + public: + using MockOsLibrary::MockOsLibrary; + std::unordered_map procMap; + void *getProcAddress(const std::string &procName) override { + if (procMap.find(procName) != procMap.end()) { + return procMap[procName]; + } else { + return getProcAddressReturn; + } + } +}; diff --git a/shared/test/common/os_interface/linux/drm_memory_manager_fixture.h b/shared/test/common/os_interface/linux/drm_memory_manager_fixture.h index b709c4719d..dd19387c12 100644 --- a/shared/test/common/os_interface/linux/drm_memory_manager_fixture.h +++ b/shared/test/common/os_interface/linux/drm_memory_manager_fixture.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2023 Intel Corporation + * Copyright (C) 2019-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -69,14 +69,14 @@ struct MockedMemoryInfo : public NEO::MemoryInfo { size_t getMemoryRegionSize(uint32_t memoryBank) override { return 1024u; } - int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) override { + int createGemExt(const MemRegionsVec &memClassInstances, size_t allocSize, uint32_t &handle, uint64_t patIndex, std::optional vmId, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, bool isUSMHostAllocation, void *addr) override { if (allocSize == 0) { return EINVAL; } handle = 1u; return 0; } - int createGemExtWithSingleRegion(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle) override { + int createGemExtWithSingleRegion(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle, bool isUSMHostAllocation, void *addr) override { if (allocSize == 0) { return EINVAL; } @@ -84,7 +84,7 @@ struct MockedMemoryInfo : public NEO::MemoryInfo { pairHandlePassed = pairHandle; return 0; } - int createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex) override { + int createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, bool isUSMHostAllocation, void *addr) override { if (allocSize == 0) { return EINVAL; } @@ -92,7 +92,7 @@ struct MockedMemoryInfo : public NEO::MemoryInfo { banks = memoryBanks; return 0; } - int createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle, bool isChunked, uint32_t numOfChunks) override { + int createGemExtWithMultipleRegions(uint32_t memoryBanks, size_t allocSize, uint32_t &handle, uint64_t patIndex, int32_t pairHandle, bool isChunked, uint32_t numOfChunks, bool isUSMHostAllocation, void *addr) override { if (allocSize == 0) { return EINVAL; } diff --git a/shared/test/common/test_files/igdrcl.config b/shared/test/common/test_files/igdrcl.config index 75fd8b6f5f..f1011e15b4 100644 --- a/shared/test/common/test_files/igdrcl.config +++ b/shared/test/common/test_files/igdrcl.config @@ -575,4 +575,6 @@ InOrderDuplicatedCounterStorageEnabled = -1 OverrideCpuCaching = -1 EnableDeviceUsmAllocationPool = -1 EnableHostUsmAllocationPool = -1 +EnableHostAllocationMemPolicy = 0 +OverrideHostAllocationMemPolicyMode = -1 # Please don't edit below this line diff --git a/shared/test/unit_test/os_interface/linux/CMakeLists.txt b/shared/test/unit_test/os_interface/linux/CMakeLists.txt index 2689af946d..b3e7adbdb2 100644 --- a/shared/test/unit_test/os_interface/linux/CMakeLists.txt +++ b/shared/test/unit_test/os_interface/linux/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright (C) 2020-2023 Intel Corporation +# Copyright (C) 2020-2024 Intel Corporation # # SPDX-License-Identifier: MIT # @@ -33,6 +33,7 @@ set(NEO_CORE_OS_INTERFACE_TESTS_LINUX ${CMAKE_CURRENT_SOURCE_DIR}/drm_uuid_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_version_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/file_logger_linux_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/numa_library_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/product_helper_uuid_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/product_helper_linux_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/product_helper_linux_tests.h diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_info_prelim_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_info_prelim_tests.cpp index 2139a53919..a4d48d0557 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_info_prelim_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_info_prelim_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022-2023 Intel Corporation + * Copyright (C) 2022-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -8,11 +8,13 @@ #include "shared/source/memory_manager/memory_banks.h" #include "shared/source/os_interface/linux/i915.h" #include "shared/source/os_interface/linux/memory_info.h" +#include "shared/source/os_interface/linux/numa_library.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/helpers/variable_backup.h" #include "shared/test/common/libult/linux/drm_mock_helper.h" #include "shared/test/common/libult/linux/drm_query_mock.h" #include "shared/test/common/mocks/mock_execution_environment.h" +#include "shared/test/common/mocks/mock_os_library.h" #include "shared/test/common/test_macros/test.h" #include "gtest/gtest.h" @@ -373,7 +375,7 @@ TEST(MemoryInfo, givenMemoryInfoWithRegionsWhenCreatingGemWithExtensionsThenRetu ASSERT_NE(nullptr, memoryInfo); uint32_t numOfChunks = 0; - auto ret = memoryInfo->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks); + auto ret = memoryInfo->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, false, nullptr); EXPECT_EQ(1u, handle); EXPECT_EQ(0, ret); EXPECT_EQ(1u, drm->ioctlCallsCount); @@ -397,7 +399,7 @@ TEST(MemoryInfo, givenMemoryInfoWithRegionsWhenCreatingGemExtWithSingleRegionThe auto memoryInfo = std::make_unique(regionInfo, *drm); ASSERT_NE(nullptr, memoryInfo); - auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, -1); + auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, -1, false, nullptr); EXPECT_EQ(1u, handle); EXPECT_EQ(0, ret); EXPECT_EQ(1u, drm->ioctlCallsCount); @@ -428,16 +430,310 @@ TEST(MemoryInfo, givenMemoryInfoWithRegionsWhenCreatingGemExtWithPairHandleThenR uint32_t pairHandle = 0; auto memoryInfo = std::make_unique(regionInfo, *drm); ASSERT_NE(nullptr, memoryInfo); - auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, pairHandle, 0, -1); + auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, pairHandle, 0, -1, false, nullptr); EXPECT_EQ(0, ret); EXPECT_EQ(1u, drm->ioctlCallsCount); uint32_t handle = 0; - ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, pairHandle); + ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, pairHandle, false, nullptr); EXPECT_EQ(0, ret); EXPECT_EQ(2u, drm->ioctlCallsCount); } +struct WhiteBoxNumaLibrary : Linux::NumaLibrary { + using Linux::NumaLibrary::numaLibNameStr; + using Linux::NumaLibrary::procGetMemPolicyStr; + using Linux::NumaLibrary::procNumaAvailableStr; + using Linux::NumaLibrary::procNumaMaxNodeStr; + using OsLibraryLoadPtr = NumaLibrary::OsLibraryLoadPtr; + using GetMemPolicyPtr = NumaLibrary::GetMemPolicyPtr; + using NumaAvailablePtr = NumaLibrary::NumaAvailablePtr; + using NumaMaxNodePtr = NumaLibrary::NumaMaxNodePtr; + using Linux::NumaLibrary::getMemPolicyFunction; + using Linux::NumaLibrary::osLibrary; + using Linux::NumaLibrary::osLibraryLoadFunction; +}; + +TEST(MemoryInfo, givenMemoryInfoWithMemoryPolicyEnabledWhenCallingCreateGemExtWithNonHostAllocationThenIoctlIsReturnedWithoutMemPolicy) { + DebugManagerStateRestore restorer; + debugManager.flags.EnableHostAllocationMemPolicy.set(1); + debugManager.flags.OverrideHostAllocationMemPolicyMode.set(-1); + std::vector regionInfo(3); + regionInfo[0].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_SYSTEM, 0}; + regionInfo[0].probedSize = 8 * MemoryConstants::gigaByte; + regionInfo[1].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}; + regionInfo[1].probedSize = 16 * MemoryConstants::gigaByte; + regionInfo[2].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 1}; + regionInfo[2].probedSize = 32 * MemoryConstants::gigaByte; + + auto executionEnvironment = std::make_unique(); + auto drm = std::make_unique(*executionEnvironment->rootDeviceEnvironments[0]); + + // setup numa library in MemoryInfo + WhiteBoxNumaLibrary::GetMemPolicyPtr memPolicyHandler = + [](int *, unsigned long[], unsigned long, void *, unsigned long) -> long { return 0; }; + WhiteBoxNumaLibrary::NumaAvailablePtr numaAvailableHandler = + [](void) -> int { return 0; }; + WhiteBoxNumaLibrary::NumaMaxNodePtr numaMaxNodeHandler = + [](void) -> int { return 4; }; + MockOsLibrary::loadLibraryNewObject = new MockOsLibraryCustom(nullptr, true); + MockOsLibraryCustom *osLibrary = static_cast(MockOsLibrary::loadLibraryNewObject); + // register proc pointers + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procGetMemPolicyStr)] = reinterpret_cast(memPolicyHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaAvailableStr)] = reinterpret_cast(numaAvailableHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaMaxNodeStr)] = reinterpret_cast(numaMaxNodeHandler); + WhiteBoxNumaLibrary::osLibraryLoadFunction = MockOsLibraryCustom::load; + auto memoryInfo = std::make_unique(regionInfo, *drm); + ASSERT_NE(nullptr, memoryInfo); + ASSERT_TRUE(memoryInfo->isMemPolicySupported()); + + uint32_t handle = 0; + MemRegionsVec memClassInstance = {regionInfo[0].region, regionInfo[1].region}; + uint32_t numOfChunks = 0; + auto ret = memoryInfo->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, false, nullptr); + EXPECT_EQ(1u, handle); + EXPECT_EQ(0, ret); + ASSERT_TRUE(drm->context.receivedCreateGemExt); + EXPECT_EQ(1024u, drm->context.receivedCreateGemExt->size); + EXPECT_EQ(std::nullopt, drm->context.receivedCreateGemExt->memPolicyExt.mode); + EXPECT_EQ(std::nullopt, drm->context.receivedCreateGemExt->memPolicyExt.nodeMask); + + MockOsLibrary::loadLibraryNewObject = nullptr; + WhiteBoxNumaLibrary::osLibrary.reset(); +} + +TEST(MemoryInfo, givenMemoryInfoWithMemoryPolicyEnabledWhenCallingCreateGemExtForHostAllocationThenIoctlIsCalledWithMemoryPolicy) { + DebugManagerStateRestore restorer; + debugManager.flags.EnableHostAllocationMemPolicy.set(1); + debugManager.flags.OverrideHostAllocationMemPolicyMode.set(-1); + std::vector regionInfo(3); + regionInfo[0].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_SYSTEM, 0}; + regionInfo[0].probedSize = 8 * MemoryConstants::gigaByte; + regionInfo[1].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}; + regionInfo[1].probedSize = 16 * MemoryConstants::gigaByte; + regionInfo[2].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 1}; + regionInfo[2].probedSize = 32 * MemoryConstants::gigaByte; + + auto executionEnvironment = std::make_unique(); + auto drm = std::make_unique(*executionEnvironment->rootDeviceEnvironments[0]); + + constexpr int num_numa = 4; + // setup numa library in MemoryInfo + WhiteBoxNumaLibrary::GetMemPolicyPtr memPolicyHandler = + [](int *mode, unsigned long nodeMask[], unsigned long, void *, unsigned long) -> long { + if (mode) { + *mode = 0; + } + for (int i = 0; i < num_numa; i++) { + nodeMask[i] = i; + } + return 0; + }; + WhiteBoxNumaLibrary::NumaAvailablePtr numaAvailableHandler = + [](void) -> int { return 0; }; + WhiteBoxNumaLibrary::NumaMaxNodePtr numaMaxNodeHandler = + [](void) -> int { return num_numa - 1; }; + MockOsLibrary::loadLibraryNewObject = new MockOsLibraryCustom(nullptr, true); + MockOsLibraryCustom *osLibrary = static_cast(MockOsLibrary::loadLibraryNewObject); + // register proc pointers + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procGetMemPolicyStr)] = reinterpret_cast(memPolicyHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaAvailableStr)] = reinterpret_cast(numaAvailableHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaMaxNodeStr)] = reinterpret_cast(numaMaxNodeHandler); + + WhiteBoxNumaLibrary::osLibraryLoadFunction = MockOsLibraryCustom::load; + + auto memoryInfo = std::make_unique(regionInfo, *drm); + ASSERT_NE(nullptr, memoryInfo); + ASSERT_TRUE(memoryInfo->isMemPolicySupported()); + + uint32_t handle = 0; + MemRegionsVec memClassInstance = {regionInfo[0].region, regionInfo[1].region}; + uint32_t numOfChunks = 0; + auto ret = memoryInfo->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, true, nullptr); + EXPECT_EQ(1u, handle); + EXPECT_EQ(0, ret); + ASSERT_TRUE(drm->context.receivedCreateGemExt); + EXPECT_EQ(1024u, drm->context.receivedCreateGemExt->size); + EXPECT_EQ(0u, drm->context.receivedCreateGemExt->memPolicyExt.mode); + EXPECT_EQ(4u, drm->context.receivedCreateGemExt->memPolicyExt.nodeMask.value().size()); + for (auto i = 0u; i < drm->context.receivedCreateGemExt->memPolicyExt.nodeMask.value().size(); i++) { + EXPECT_EQ(i, drm->context.receivedCreateGemExt->memPolicyExt.nodeMask.value()[i]); + } + MockOsLibrary::loadLibraryNewObject = nullptr; + WhiteBoxNumaLibrary::osLibrary.reset(); +} + +TEST(MemoryInfo, givenMemoryInfoWithMemoryPolicyEnabledAndOverrideMemoryPolicyModeWhenCallingCreateGemExtForHostAllocationThenIoctlIsCalledWithMemoryPolicy) { + DebugManagerStateRestore restorer; + debugManager.flags.EnableHostAllocationMemPolicy.set(1); + debugManager.flags.OverrideHostAllocationMemPolicyMode.set(0); + std::vector regionInfo(3); + regionInfo[0].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_SYSTEM, 0}; + regionInfo[0].probedSize = 8 * MemoryConstants::gigaByte; + regionInfo[1].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}; + regionInfo[1].probedSize = 16 * MemoryConstants::gigaByte; + regionInfo[2].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 1}; + regionInfo[2].probedSize = 32 * MemoryConstants::gigaByte; + + auto executionEnvironment = std::make_unique(); + auto drm = std::make_unique(*executionEnvironment->rootDeviceEnvironments[0]); + + constexpr int num_numa = 4; + // setup numa library in MemoryInfo + WhiteBoxNumaLibrary::GetMemPolicyPtr memPolicyHandler = + [](int *mode, unsigned long nodeMask[], unsigned long, void *, unsigned long) -> long { + if (mode) { + *mode = 3; + } + for (int i = 0; i < num_numa; i++) { + nodeMask[i] = i; + } + return 0; + }; + WhiteBoxNumaLibrary::NumaAvailablePtr numaAvailableHandler = + [](void) -> int { return 0; }; + WhiteBoxNumaLibrary::NumaMaxNodePtr numaMaxNodeHandler = + [](void) -> int { return num_numa - 1; }; + MockOsLibrary::loadLibraryNewObject = new MockOsLibraryCustom(nullptr, true); + MockOsLibraryCustom *osLibrary = static_cast(MockOsLibrary::loadLibraryNewObject); + // register proc pointers + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procGetMemPolicyStr)] = reinterpret_cast(memPolicyHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaAvailableStr)] = reinterpret_cast(numaAvailableHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaMaxNodeStr)] = reinterpret_cast(numaMaxNodeHandler); + + WhiteBoxNumaLibrary::osLibraryLoadFunction = MockOsLibraryCustom::load; + + auto memoryInfo = std::make_unique(regionInfo, *drm); + ASSERT_NE(nullptr, memoryInfo); + ASSERT_TRUE(memoryInfo->isMemPolicySupported()); + + uint32_t handle = 0; + MemRegionsVec memClassInstance = {regionInfo[0].region, regionInfo[1].region}; + uint32_t numOfChunks = 0; + auto ret = memoryInfo->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, true, nullptr); + EXPECT_EQ(1u, handle); + EXPECT_EQ(0, ret); + ASSERT_TRUE(drm->context.receivedCreateGemExt); + EXPECT_EQ(1024u, drm->context.receivedCreateGemExt->size); + EXPECT_EQ(0u, drm->context.receivedCreateGemExt->memPolicyExt.mode); + EXPECT_EQ(4u, drm->context.receivedCreateGemExt->memPolicyExt.nodeMask.value().size()); + for (auto i = 0u; i < drm->context.receivedCreateGemExt->memPolicyExt.nodeMask.value().size(); i++) { + EXPECT_EQ(i, drm->context.receivedCreateGemExt->memPolicyExt.nodeMask.value()[i]); + } + + MockOsLibrary::loadLibraryNewObject = nullptr; + WhiteBoxNumaLibrary::osLibrary.reset(); +} + +TEST(MemoryInfo, givenMemoryInfoWithMemoryPolicyEnabledWhenCallingCreateGemExtWithIncorrectGetMemPolicyHandlerThenIoctlIsReturnedWithoutMemPolicy) { + DebugManagerStateRestore restorer; + debugManager.flags.EnableHostAllocationMemPolicy.set(1); + debugManager.flags.OverrideHostAllocationMemPolicyMode.set(-1); + std::vector regionInfo(3); + regionInfo[0].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_SYSTEM, 0}; + regionInfo[0].probedSize = 8 * MemoryConstants::gigaByte; + regionInfo[1].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}; + regionInfo[1].probedSize = 16 * MemoryConstants::gigaByte; + regionInfo[2].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 1}; + regionInfo[2].probedSize = 32 * MemoryConstants::gigaByte; + + auto executionEnvironment = std::make_unique(); + auto drm = std::make_unique(*executionEnvironment->rootDeviceEnvironments[0]); + + // setup numa library in MemoryInfo + WhiteBoxNumaLibrary::GetMemPolicyPtr memPolicyHandler = + [](int *, unsigned long[], unsigned long, void *, unsigned long) -> long { return -1; }; + WhiteBoxNumaLibrary::NumaAvailablePtr numaAvailableHandler = + [](void) -> int { return 0; }; + WhiteBoxNumaLibrary::NumaMaxNodePtr numaMaxNodeHandler = + [](void) -> int { return 4; }; + MockOsLibrary::loadLibraryNewObject = new MockOsLibraryCustom(nullptr, true); + MockOsLibraryCustom *osLibrary = static_cast(MockOsLibrary::loadLibraryNewObject); + // register proc pointers + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procGetMemPolicyStr)] = reinterpret_cast(memPolicyHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaAvailableStr)] = reinterpret_cast(numaAvailableHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaMaxNodeStr)] = reinterpret_cast(numaMaxNodeHandler); + WhiteBoxNumaLibrary::osLibraryLoadFunction = MockOsLibraryCustom::load; + auto memoryInfo = std::make_unique(regionInfo, *drm); + ASSERT_NE(nullptr, memoryInfo); + ASSERT_TRUE(memoryInfo->isMemPolicySupported()); + + uint32_t handle = 0; + MemRegionsVec memClassInstance = {regionInfo[0].region, regionInfo[1].region}; + uint32_t numOfChunks = 0; + auto ret = memoryInfo->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, true, nullptr); + EXPECT_EQ(1u, handle); + EXPECT_EQ(0, ret); + ASSERT_TRUE(drm->context.receivedCreateGemExt); + EXPECT_EQ(1024u, drm->context.receivedCreateGemExt->size); + EXPECT_EQ(std::nullopt, drm->context.receivedCreateGemExt->memPolicyExt.mode); + EXPECT_EQ(std::nullopt, drm->context.receivedCreateGemExt->memPolicyExt.nodeMask); + + MockOsLibrary::loadLibraryNewObject = nullptr; + WhiteBoxNumaLibrary::osLibrary.reset(); +} + +TEST(MemoryInfo, givenMemoryInfoWithMemoryPolicyEnabledAndInvalidOsLibraryWhenCallingInitializingNumaLibraryThenMemPolicyIsNotSupported) { + DebugManagerStateRestore restorer; + debugManager.flags.EnableHostAllocationMemPolicy.set(1); + debugManager.flags.OverrideHostAllocationMemPolicyMode.set(-1); + std::vector regionInfo(3); + regionInfo[0].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_SYSTEM, 0}; + regionInfo[0].probedSize = 8 * MemoryConstants::gigaByte; + regionInfo[1].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}; + regionInfo[1].probedSize = 16 * MemoryConstants::gigaByte; + regionInfo[2].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 1}; + regionInfo[2].probedSize = 32 * MemoryConstants::gigaByte; + + auto executionEnvironment = std::make_unique(); + auto drm = std::make_unique(*executionEnvironment->rootDeviceEnvironments[0]); + + MockOsLibrary::loadLibraryNewObject = nullptr; + WhiteBoxNumaLibrary::osLibraryLoadFunction = MockOsLibraryCustom::load; + auto memoryInfo = std::make_unique(regionInfo, *drm); + ASSERT_NE(nullptr, memoryInfo); + ASSERT_FALSE(memoryInfo->isMemPolicySupported()); + + MockOsLibrary::loadLibraryNewObject = nullptr; + WhiteBoxNumaLibrary::osLibrary.reset(); +} + +TEST(MemoryInfo, givenMemoryInfoWithMemoryPolicyDisabledAndValidOsLibraryWhenCallingInitializingNumaLibraryThenMemPolicyIsNotSupported) { + DebugManagerStateRestore restorer; + debugManager.flags.EnableHostAllocationMemPolicy.set(0); + debugManager.flags.OverrideHostAllocationMemPolicyMode.set(-1); + std::vector regionInfo(3); + regionInfo[0].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_SYSTEM, 0}; + regionInfo[0].probedSize = 8 * MemoryConstants::gigaByte; + regionInfo[1].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}; + regionInfo[1].probedSize = 16 * MemoryConstants::gigaByte; + regionInfo[2].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 1}; + regionInfo[2].probedSize = 32 * MemoryConstants::gigaByte; + + auto executionEnvironment = std::make_unique(); + auto drm = std::make_unique(*executionEnvironment->rootDeviceEnvironments[0]); + + // setup numa library in MemoryInfo + WhiteBoxNumaLibrary::GetMemPolicyPtr memPolicyHandler = + [](int *, unsigned long[], unsigned long, void *, unsigned long) -> long { return -1; }; + WhiteBoxNumaLibrary::NumaAvailablePtr numaAvailableHandler = + [](void) -> int { return 0; }; + WhiteBoxNumaLibrary::NumaMaxNodePtr numaMaxNodeHandler = + [](void) -> int { return 4; }; + MockOsLibrary::loadLibraryNewObject = new MockOsLibraryCustom(nullptr, true); + MockOsLibraryCustom *osLibrary = static_cast(MockOsLibrary::loadLibraryNewObject); + // register proc pointers + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procGetMemPolicyStr)] = reinterpret_cast(memPolicyHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaAvailableStr)] = reinterpret_cast(numaAvailableHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaMaxNodeStr)] = reinterpret_cast(numaMaxNodeHandler); + WhiteBoxNumaLibrary::osLibraryLoadFunction = MockOsLibraryCustom::load; + auto memoryInfo = std::make_unique(regionInfo, *drm); + ASSERT_NE(nullptr, memoryInfo); + ASSERT_FALSE(memoryInfo->isMemPolicySupported()); + + MockOsLibrary::loadLibraryNewObject = nullptr; + WhiteBoxNumaLibrary::osLibrary.reset(); +} TEST(MemoryInfo, givenMemoryInfoWithRegionsWhenCreatingGemExtWithChunkingButSizeLessThanAllowedThenExceptionIsThrown) { DebugManagerStateRestore restorer; debugManager.flags.EnableLocalMemory.set(1); @@ -461,7 +757,7 @@ TEST(MemoryInfo, givenMemoryInfoWithRegionsWhenCreatingGemExtWithChunkingButSize bool isChunked = true; auto memoryInfo = std::make_unique(regionInfo, *drm); ASSERT_NE(nullptr, memoryInfo); - EXPECT_THROW(memoryInfo->createGemExtWithMultipleRegions(1, allocSize, handle, 0, pairHandle, isChunked, numOfChunks), std::runtime_error); + EXPECT_THROW(memoryInfo->createGemExtWithMultipleRegions(1, allocSize, handle, 0, pairHandle, isChunked, numOfChunks, false, nullptr), std::runtime_error); } TEST(MemoryInfo, givenMemoryInfoWithRegionsWhenCreatingGemExtWithChunkingWithSizeGreaterThanAllowedThenAllocationIsCreatedWithChunking) { @@ -487,7 +783,7 @@ TEST(MemoryInfo, givenMemoryInfoWithRegionsWhenCreatingGemExtWithChunkingWithSiz bool isChunked = true; auto memoryInfo = std::make_unique(regionInfo, *drm); ASSERT_NE(nullptr, memoryInfo); - auto ret = memoryInfo->createGemExtWithMultipleRegions(1, allocSize, handle, 0, pairHandle, isChunked, numOfChunks); + auto ret = memoryInfo->createGemExtWithMultipleRegions(1, allocSize, handle, 0, pairHandle, isChunked, numOfChunks, false, nullptr); EXPECT_EQ(0, ret); EXPECT_EQ(1u, drm->ioctlCallsCount); } @@ -511,7 +807,7 @@ TEST(MemoryInfo, givenMemoryInfoWithRegionsAndPrivateBOSupportWhenCreatingGemExt ASSERT_NE(nullptr, memoryInfo); uint32_t handle = 0; - auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, -1); + auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, -1, false, nullptr); EXPECT_EQ(1u, handle); EXPECT_EQ(0, ret); EXPECT_EQ(1u, drm->ioctlCallsCount); @@ -541,7 +837,7 @@ TEST(MemoryInfo, givenMemoryInfoWithRegionsAndNoPrivateBOSupportWhenCreatingGemE ASSERT_NE(nullptr, memoryInfo); uint32_t handle = 0; - auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, -1); + auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, -1, false, nullptr); EXPECT_EQ(1u, handle); EXPECT_EQ(0, ret); EXPECT_EQ(1u, drm->ioctlCallsCount); @@ -570,7 +866,7 @@ TEST(MemoryInfo, givenMemoryInfoWithRegionsAndPrivateBOSupportedAndIsPerContextV ASSERT_NE(nullptr, memoryInfo); uint32_t handle = 0; - auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, -1); + auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, -1, false, nullptr); EXPECT_EQ(1u, handle); EXPECT_EQ(0, ret); EXPECT_EQ(1u, drm->ioctlCallsCount); @@ -603,7 +899,7 @@ TEST(MemoryInfo, givenMemoryInfoWithRegionsWhenCreatingGemExtWithMultipleRegions ASSERT_NE(nullptr, memoryInfo); uint32_t handle = 0; uint32_t memoryRegions = 0b1011; - auto ret = memoryInfo->createGemExtWithMultipleRegions(memoryRegions, 1024, handle, 0); + auto ret = memoryInfo->createGemExtWithMultipleRegions(memoryRegions, 1024, handle, 0, false, nullptr); EXPECT_EQ(1u, handle); EXPECT_EQ(0, ret); EXPECT_EQ(1u, drm->ioctlCallsCount); @@ -644,7 +940,7 @@ TEST(MemoryInfo, givenMemoryInfoWithRegionsWhenCallingCreatingGemExtWithMultiple uint32_t handle = 0; uint32_t memoryRegions = 0b1011; uint32_t numOfChunks = 2; - EXPECT_THROW(memoryInfo->createGemExtWithMultipleRegions(memoryRegions, MemoryConstants::chunkThreshold / (numOfChunks * 2), handle, 0, -1, true, numOfChunks), std::runtime_error); + EXPECT_THROW(memoryInfo->createGemExtWithMultipleRegions(memoryRegions, MemoryConstants::chunkThreshold / (numOfChunks * 2), handle, 0, -1, true, numOfChunks, false, nullptr), std::runtime_error); } TEST(MemoryInfo, givenMemoryInfoWithRegionsWhenCallingCreatingGemExtWithMultipleRegionsAndChunkingThenReturnCorrectValues) { @@ -672,7 +968,7 @@ TEST(MemoryInfo, givenMemoryInfoWithRegionsWhenCallingCreatingGemExtWithMultiple uint32_t memoryRegions = 0b1011; uint32_t numOfChunks = 2; size_t size = MemoryConstants::chunkThreshold * numOfChunks; - auto ret = memoryInfo->createGemExtWithMultipleRegions(memoryRegions, size, handle, 0, -1, true, numOfChunks); + auto ret = memoryInfo->createGemExtWithMultipleRegions(memoryRegions, size, handle, 0, -1, true, numOfChunks, false, nullptr); EXPECT_EQ(1u, handle); EXPECT_EQ(0, ret); EXPECT_EQ(1u, drm->ioctlCallsCount); diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_info_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_info_tests.cpp index 3bbfc308eb..aa7c2484d7 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_info_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_info_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2023 Intel Corporation + * Copyright (C) 2019-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -236,7 +236,7 @@ HWTEST2_F(MemoryInfoTest, givenMemoryInfoWithRegionsWhenCreatingGemWithExtension uint32_t handle = 0; MemRegionsVec memClassInstance = {regionInfo[0].region, regionInfo[1].region}; uint32_t numOfChunks = 0; - auto ret = memoryInfo->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks); + auto ret = memoryInfo->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, false, nullptr); EXPECT_EQ(1u, handle); EXPECT_EQ(0, ret); EXPECT_EQ(1u, drm->ioctlCallsCount); @@ -259,7 +259,7 @@ HWTEST2_F(MemoryInfoTest, givenMemoryInfoWithRegionsWhenCreatingGemExtWithSingle auto memoryInfo = std::make_unique(regionInfo, *drm); ASSERT_NE(nullptr, memoryInfo); - auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, -1); + auto ret = memoryInfo->createGemExtWithSingleRegion(1, 1024, handle, 0, -1, false, nullptr); EXPECT_EQ(1u, handle); EXPECT_EQ(0, ret); EXPECT_EQ(1u, drm->ioctlCallsCount); diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_localmem_prelim_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_localmem_prelim_tests.cpp index c5365346e1..bcf6820e39 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_localmem_prelim_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_localmem_prelim_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022-2023 Intel Corporation + * Copyright (C) 2022-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -62,6 +62,7 @@ TEST_F(DrmMemoryManagerLocalMemoryPrelimTest, givenDrmMemoryManagerWithPrelimSup (1 << (MemoryBanks::getBankForLocalMemory(0) - 1)), 1, -1, + false, false)); ASSERT_NE(nullptr, bo); @@ -1982,6 +1983,7 @@ TEST_F(DrmMemoryManagerLocalMemoryPrelimTest, givenPrintBOCreateDestroyResultFla (1 << (MemoryBanks::getBankForLocalMemory(0) - 1)), 1, -1, + false, false)); EXPECT_NE(nullptr, bo); diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_localmem_upstream_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_localmem_upstream_tests.cpp index f7513cde54..681598faef 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_localmem_upstream_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_localmem_upstream_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2023 Intel Corporation + * Copyright (C) 2020-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -54,7 +54,7 @@ class DrmMemoryManagerFixtureImpl : public DrmMemoryManagerFixture { std::unique_ptr> backup; }; -BufferObject *createBufferObjectInMemoryRegion(Drm *drm, Gmm *gmm, AllocationType allocationType, uint64_t gpuAddress, size_t size, uint32_t memoryBanks, size_t maxOsContextCount, bool isSystemMemoryPool); +BufferObject *createBufferObjectInMemoryRegion(Drm *drm, Gmm *gmm, AllocationType allocationType, uint64_t gpuAddress, size_t size, uint32_t memoryBanks, size_t maxOsContextCount, bool isSystemMemoryPool, bool isHostUSMAllocation); class DrmMemoryManagerLocalMemoryTest : public ::testing::Test { public: @@ -127,6 +127,7 @@ HWTEST2_F(DrmMemoryManagerLocalMemoryTest, givenDrmMemoryManagerWhenCreateBuffer (1 << (MemoryBanks::getBankForLocalMemory(0) - 1)), 1, -1, + false, false)); ASSERT_NE(nullptr, bo); EXPECT_EQ(1u, mock->ioctlCallsCount); @@ -449,7 +450,7 @@ class DrmMemoryManagerLocalMemoryMemoryBankMock : public TestedDrmMemoryManager uint32_t memoryBanks, size_t maxOsContextCount, int32_t pairHandle, - bool isSystemMemoryPool) override { + bool isSystemMemoryPool, bool isUSMHostAllocation) override { memoryBankIsOne = (memoryBanks == 1) ? true : false; return nullptr; } diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index fe992e70ce..54b3edd73a 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022-2023 Intel Corporation + * Copyright (C) 2022-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -5459,7 +5459,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenDrmMemor auto gpuAddress = 0x1234u; auto size = MemoryConstants::pageSize; - auto bo = std::unique_ptr(memoryManager->createBufferObjectInMemoryRegion(rootDeviceIndex, nullptr, AllocationType::buffer, gpuAddress, size, MemoryBanks::mainBank, 1, -1, false)); + auto bo = std::unique_ptr(memoryManager->createBufferObjectInMemoryRegion(rootDeviceIndex, nullptr, AllocationType::buffer, gpuAddress, size, MemoryBanks::mainBank, 1, -1, false, false)); EXPECT_EQ(nullptr, bo); } @@ -5467,7 +5467,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenDrmMemor auto gpuAddress = 0x1234u; auto size = 0u; - auto bo = std::unique_ptr(memoryManager->createBufferObjectInMemoryRegion(rootDeviceIndex, nullptr, AllocationType::buffer, gpuAddress, size, MemoryBanks::mainBank, 1, -1, false)); + auto bo = std::unique_ptr(memoryManager->createBufferObjectInMemoryRegion(rootDeviceIndex, nullptr, AllocationType::buffer, gpuAddress, size, MemoryBanks::mainBank, 1, -1, false, false)); EXPECT_EQ(nullptr, bo); } diff --git a/shared/test/unit_test/os_interface/linux/drm_with_prelim_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_with_prelim_tests.cpp index 844a033fb5..9f3b61f0e6 100644 --- a/shared/test/unit_test/os_interface/linux/drm_with_prelim_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_with_prelim_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Intel Corporation + * Copyright (C) 2021-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -137,7 +137,7 @@ TEST_F(IoctlHelperPrelimFixture, givenPrelimsWhenCreateGemExtThenReturnSuccess) uint32_t handle = 0; MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; uint32_t numOfChunks = 0; - auto ret = ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks); + auto ret = ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, std::nullopt, std::nullopt); EXPECT_EQ(1u, handle); EXPECT_EQ(0, ret); @@ -161,7 +161,7 @@ TEST_F(IoctlHelperPrelimFixture, givenPrelimsWhenCreateGemExtWithChunkingThenGet uint32_t handle = 0; uint32_t getNumOfChunks = 2; MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; - ioctlHelper->createGemExt(memClassInstance, allocSize, handle, 0, {}, -1, true, getNumOfChunks); + ioctlHelper->createGemExt(memClassInstance, allocSize, handle, 0, {}, -1, true, getNumOfChunks, std::nullopt, std::nullopt); std::string output = testing::internal::GetCapturedStdout(); std::string expectedOutput("GEM_CREATE_EXT BO-1 with BOChunkingSize 65536, chunkingParamRegion.param.data 65536, numOfChunks 2\n"); EXPECT_EQ(expectedOutput, output); @@ -178,7 +178,7 @@ TEST_F(IoctlHelperPrelimFixture, givenPrelimsWhenCreateGemExtWithChunkingAndAllo uint32_t handle = 0; uint32_t getNumOfChunks = 2; MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; - EXPECT_THROW(ioctlHelper->createGemExt(memClassInstance, allocSize, handle, 0, {}, -1, true, getNumOfChunks), std::runtime_error); + EXPECT_THROW(ioctlHelper->createGemExt(memClassInstance, allocSize, handle, 0, {}, -1, true, getNumOfChunks, std::nullopt, std::nullopt), std::runtime_error); } TEST_F(IoctlHelperPrelimFixture, givenPrelimsWhenCreateGemExtWithDebugFlagThenPrintDebugInfo) { @@ -190,7 +190,7 @@ TEST_F(IoctlHelperPrelimFixture, givenPrelimsWhenCreateGemExtWithDebugFlagThenPr uint32_t handle = 0; MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; uint32_t numOfChunks = 0; - ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks); + ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, std::nullopt, std::nullopt); std::string output = testing::internal::GetCapturedStdout(); std::string expectedOutput("Performing GEM_CREATE_EXT with { size: 1024, param: 0x1000000010001, memory class: 1, memory instance: 0 }\nGEM_CREATE_EXT has returned: 0 BO-1 with size: 1024\n"); diff --git a/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_dg1.cpp b/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_dg1.cpp index 0cfcec9269..33a2e89e01 100644 --- a/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_dg1.cpp +++ b/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_dg1.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Intel Corporation + * Copyright (C) 2021-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -28,7 +28,7 @@ DG1TEST_F(IoctlHelperTestsDg1, givenDg1WhenCreateGemExtThenReturnCorrectValue) { uint32_t handle = 0; MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; uint32_t numOfChunks = 0; - auto ret = ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks); + auto ret = ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, std::nullopt, std::nullopt); EXPECT_EQ(0, ret); EXPECT_EQ(1u, handle); @@ -50,7 +50,7 @@ DG1TEST_F(IoctlHelperTestsDg1, givenDg1WithDrmTipWhenCreateGemExtWithDebugFlagTh uint32_t handle = 0; MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; uint32_t numOfChunks = 0; - auto ret = ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks); + auto ret = ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, std::nullopt, std::nullopt); std::string output = testing::internal::GetCapturedStdout(); std::string expectedOutput("Performing GEM_CREATE_EXT with { size: 1024, memory class: 1, memory instance: 0 }\nGEM_CREATE_EXT with EXT_MEMORY_REGIONS has returned: 0 BO-1 with size: 1024\n"); @@ -72,7 +72,7 @@ DG1TEST_F(IoctlHelperTestsDg1, givenDg1WhenCreateGemExtWithDebugFlagThenPrintDeb uint32_t handle = 0; MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; uint32_t numOfChunks = 0; - auto ret = ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks); + auto ret = ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, std::nullopt, std::nullopt); std::string output = testing::internal::GetCapturedStdout(); std::string expectedOutput("Performing GEM_CREATE_EXT with { size: 1024, memory class: 1, memory instance: 0 }\nGEM_CREATE_EXT with EXT_MEMORY_REGIONS has returned: -1 BO-0 with size: 1024\nGEM_CREATE_EXT with EXT_SETPARAM has returned: 0 BO-1 with size: 1024\n"); diff --git a/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_prelim.cpp b/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_prelim.cpp index 371e256dfc..8064329038 100644 --- a/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_prelim.cpp +++ b/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_prelim.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Intel Corporation + * Copyright (C) 2021-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -366,8 +366,7 @@ TEST_F(IoctlPrelimHelperTests, givenPrelimWhenGettingEuStallPropertiesThenCorrec EXPECT_EQ(properties[6], prelim_drm_i915_eu_stall_property_id::PRELIM_DRM_I915_EU_STALL_PROP_ENGINE_CLASS); EXPECT_EQ(properties[7], prelim_drm_i915_gem_engine_class::PRELIM_I915_ENGINE_CLASS_COMPUTE); EXPECT_EQ(properties[8], prelim_drm_i915_eu_stall_property_id::PRELIM_DRM_I915_EU_STALL_PROP_ENGINE_INSTANCE); - EXPECT_EQ(properties[9], 1u); - EXPECT_EQ(properties[10], prelim_drm_i915_eu_stall_property_id::PRELIM_DRM_I915_EU_STALL_PROP_EVENT_REPORT_COUNT); + EXPECT_EQ(properties[11], 20u); } @@ -375,6 +374,98 @@ TEST_F(IoctlPrelimHelperTests, givenPrelimWhenGettingEuStallFdParameterThenCorre EXPECT_EQ(static_cast(PRELIM_I915_PERF_FLAG_FD_EU_STALL), ioctlHelper.getEuStallFdParameter()); } +struct MockIoctlHelperPrelim20 : IoctlHelperPrelim20 { + using IoctlHelperPrelim20::createGemExt; + using IoctlHelperPrelim20::IoctlHelperPrelim20; + int ioctl(DrmIoctl request, void *arg) override { + ioctlCallCount++; + if (request == DrmIoctl::gemCreateExt) { + lastGemCreateContainedMemPolicy = checkWhetherGemCreateExtContainsMemPolicy(arg); + if (overrideGemCreateExtReturnValue.has_value()) { + return *overrideGemCreateExtReturnValue; + } + } + return IoctlHelperPrelim20::ioctl(request, arg); + } + bool checkWhetherGemCreateExtContainsMemPolicy(void *arg) { + auto &gemCreateExt = *reinterpret_cast(arg); + auto pExtensionBase = reinterpret_cast(gemCreateExt.extensions); + while (pExtensionBase != nullptr) { + if (pExtensionBase->name == PRELIM_I915_GEM_CREATE_EXT_MEMORY_POLICY) { + auto lastPolicy = reinterpret_cast(pExtensionBase); + lastPolicyMode = lastPolicy->mode; + lastPolicyFlags = lastPolicy->flags; + lastPolicyNodeMask.clear(); + auto nodeMaskPtr = reinterpret_cast(lastPolicy->nodemask_ptr); + for (auto i = 0u; i < lastPolicy->nodemask_max; i++) { + lastPolicyNodeMask.push_back(nodeMaskPtr[i]); + } + return true; + } + pExtensionBase = reinterpret_cast(pExtensionBase->next_extension); + } + return false; + } + size_t ioctlCallCount = 0; + bool lastGemCreateContainedMemPolicy = false; + std::optional overrideGemCreateExtReturnValue{}; + uint32_t lastPolicyMode = 0; + uint32_t lastPolicyFlags = 0; + std::vector lastPolicyNodeMask{}; +}; + +TEST(IoctlPrelimHelperCreateGemExtTests, givenPrelimWhenCreateGemExtWithMemPolicyThenMemPolicyExtensionsIsAdded) { + DebugManagerStateRestore stateRestore; + debugManager.flags.PrintBOCreateDestroyResult.set(true); + testing::internal::CaptureStdout(); + + auto executionEnvironment = std::make_unique(); + auto drm = std::make_unique(*executionEnvironment->rootDeviceEnvironments[0]); + MockIoctlHelperPrelim20 mockIoctlHelper{*drm}; + + uint32_t handle = 0; + MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_SYSTEM, 0}}; + uint32_t numOfChunks = 0; + std::vector memPolicy; + memPolicy.push_back(1); + uint32_t memPolicyMode = 0; + mockIoctlHelper.overrideGemCreateExtReturnValue = 0; + mockIoctlHelper.initialize(); + auto ret = mockIoctlHelper.createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, memPolicyMode, memPolicy); + + std::string output = testing::internal::GetCapturedStdout(); + std::string expectedSubstring("memory policy:"); + EXPECT_EQ(0, ret); + EXPECT_TRUE(mockIoctlHelper.lastGemCreateContainedMemPolicy); + EXPECT_EQ(0u, mockIoctlHelper.lastPolicyFlags); + EXPECT_EQ(memPolicyMode, mockIoctlHelper.lastPolicyMode); + EXPECT_EQ(memPolicy, mockIoctlHelper.lastPolicyNodeMask); + EXPECT_TRUE(output.find(expectedSubstring) != std::string::npos); +} + +TEST(IoctlPrelimHelperCreateGemExtTests, givenPrelimWhenCreateGemExtWithMemPolicyAndChunkingThenMemPolicyExtensionsIsAdded) { + auto executionEnvironment = std::make_unique(); + auto drm = std::make_unique(*executionEnvironment->rootDeviceEnvironments[0]); + MockIoctlHelperPrelim20 mockIoctlHelper{*drm}; + + uint32_t handle = 0; + MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_SYSTEM, 0}}; + uint32_t numOfChunks = 2; + size_t size = 4 * MemoryConstants::pageSize64k; + std::vector memPolicy; + memPolicy.push_back(1); + uint32_t memPolicyMode = 0; + mockIoctlHelper.overrideGemCreateExtReturnValue = 0; + mockIoctlHelper.initialize(); + auto ret = mockIoctlHelper.createGemExt(memClassInstance, size, handle, 0, {}, -1, true, numOfChunks, memPolicyMode, memPolicy); + + EXPECT_EQ(0, ret); + EXPECT_TRUE(mockIoctlHelper.lastGemCreateContainedMemPolicy); + EXPECT_EQ(0u, mockIoctlHelper.lastPolicyFlags); + EXPECT_EQ(memPolicyMode, mockIoctlHelper.lastPolicyMode); + EXPECT_EQ(memPolicy, mockIoctlHelper.lastPolicyNodeMask); +} + class DrmMockIoctl : public DrmMock { public: DrmMockIoctl(RootDeviceEnvironment &rootDeviceEnvironment) : DrmMock(rootDeviceEnvironment) { diff --git a/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_upstream.cpp b/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_upstream.cpp index 2d48bb4579..1a4b2ea5e0 100644 --- a/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_upstream.cpp +++ b/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_upstream.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Intel Corporation + * Copyright (C) 2021-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -328,7 +328,7 @@ TEST(IoctlHelperTestsUpstream, givenUpstreamWhenCreateGemExtThenReturnCorrectVal uint32_t handle = 0; MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; uint32_t numOfChunks = 0; - auto ret = ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks); + auto ret = ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, std::nullopt, std::nullopt); EXPECT_EQ(0, ret); EXPECT_EQ(1u, handle); @@ -350,7 +350,7 @@ TEST(IoctlHelperTestsUpstream, givenUpstreamWhenCreateGemExtWithDebugFlagThenPri uint32_t handle = 0; MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; uint32_t numOfChunks = 0; - ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks); + ioctlHelper->createGemExt(memClassInstance, 1024, handle, 0, {}, -1, false, numOfChunks, std::nullopt, std::nullopt); std::string output = testing::internal::GetCapturedStdout(); std::string expectedOutput("Performing GEM_CREATE_EXT with { size: 1024, memory class: 1, memory instance: 0 }\nGEM_CREATE_EXT with EXT_MEMORY_REGIONS has returned: 0 BO-1 with size: 1024\n"); @@ -365,13 +365,13 @@ TEST(IoctlHelperTestsUpstream, givenSetPatSupportedWhenCreateGemExtThenSetPatExt uint32_t handle = 0; MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; mockIoctlHelper.isSetPatSupported = false; - auto ret = mockIoctlHelper.createGemExt(memClassInstance, 1, handle, 0, {}, -1, false, 0); + auto ret = mockIoctlHelper.createGemExt(memClassInstance, 1, handle, 0, {}, -1, false, 0, std::nullopt, std::nullopt); EXPECT_EQ(0, ret); EXPECT_EQ(1u, mockIoctlHelper.ioctlCallCount); EXPECT_FALSE(mockIoctlHelper.lastGemCreateContainedSetPat); mockIoctlHelper.isSetPatSupported = true; - ret = mockIoctlHelper.createGemExt(memClassInstance, 1, handle, 0, {}, -1, false, 0); + ret = mockIoctlHelper.createGemExt(memClassInstance, 1, handle, 0, {}, -1, false, 0, std::nullopt, std::nullopt); EXPECT_EQ(0, ret); EXPECT_EQ(2u, mockIoctlHelper.ioctlCallCount); EXPECT_TRUE(mockIoctlHelper.lastGemCreateContainedSetPat); @@ -386,13 +386,13 @@ TEST(IoctlHelperTestsUpstream, givenInvalidPatIndexWhenCreateGemExtThenSetPatExt mockIoctlHelper.isSetPatSupported = true; uint64_t invalidPatIndex = CommonConstants::unsupportedPatIndex; MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; - auto ret = mockIoctlHelper.createGemExt(memClassInstance, 1, handle, invalidPatIndex, {}, -1, false, 0); + auto ret = mockIoctlHelper.createGemExt(memClassInstance, 1, handle, invalidPatIndex, {}, -1, false, 0, std::nullopt, std::nullopt); EXPECT_EQ(0, ret); EXPECT_EQ(1u, mockIoctlHelper.ioctlCallCount); EXPECT_FALSE(mockIoctlHelper.lastGemCreateContainedSetPat); invalidPatIndex = static_cast(std::numeric_limits::max()) + 1; - ret = mockIoctlHelper.createGemExt(memClassInstance, 1, handle, invalidPatIndex, {}, -1, false, 0); + ret = mockIoctlHelper.createGemExt(memClassInstance, 1, handle, invalidPatIndex, {}, -1, false, 0, std::nullopt, std::nullopt); EXPECT_EQ(0, ret); EXPECT_EQ(2u, mockIoctlHelper.ioctlCallCount); EXPECT_FALSE(mockIoctlHelper.lastGemCreateContainedSetPat); @@ -414,7 +414,7 @@ TEST(IoctlHelperTestsUpstream, givenSetPatSupportedWhenCreateGemExtWithDebugFlag MemRegionsVec memClassInstance = {{drm_i915_gem_memory_class::I915_MEMORY_CLASS_DEVICE, 0}}; uint32_t numOfChunks = 0; uint64_t patIndex = 5; - mockIoctlHelper.createGemExt(memClassInstance, 1024, handle, patIndex, {}, -1, false, numOfChunks); + mockIoctlHelper.createGemExt(memClassInstance, 1024, handle, patIndex, {}, -1, false, numOfChunks, std::nullopt, std::nullopt); std::string output = testing::internal::GetCapturedStdout(); std::string expectedOutput("Performing GEM_CREATE_EXT with { size: 1024, memory class: 1, memory instance: 0, pat index: 5 }\nGEM_CREATE_EXT with EXT_MEMORY_REGIONS with EXT_SET_PAT has returned: 0 BO-1 with size: 1024\n"); diff --git a/shared/test/unit_test/os_interface/linux/numa_library_tests.cpp b/shared/test/unit_test/os_interface/linux/numa_library_tests.cpp new file mode 100644 index 0000000000..d64dff2127 --- /dev/null +++ b/shared/test/unit_test/os_interface/linux/numa_library_tests.cpp @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/helpers/string.h" +#include "shared/source/os_interface/linux/numa_library.h" +#include "shared/test/common/mocks/mock_os_library.h" +#include "shared/test/common/test_macros/test.h" + +#include + +using namespace NEO; +using namespace NEO::Linux; + +struct WhiteBoxNumaLibrary : NumaLibrary { + public: + using NumaLibrary::numaLibNameStr; + using NumaLibrary::procGetMemPolicyStr; + using NumaLibrary::procNumaAvailableStr; + using NumaLibrary::procNumaMaxNodeStr; + using OsLibraryLoadPtr = NumaLibrary::OsLibraryLoadPtr; + using GetMemPolicyPtr = NumaLibrary::GetMemPolicyPtr; + using NumaAvailablePtr = NumaLibrary::NumaAvailablePtr; + using NumaMaxNodePtr = NumaLibrary::NumaMaxNodePtr; + using NumaLibrary::getMemPolicyFunction; + using NumaLibrary::osLibrary; + using NumaLibrary::osLibraryLoadFunction; +}; + +TEST(NumaLibraryTests, givenNumaLibraryWithValidMockOsLibraryWhenCallingGetMemPolicyThenZeroIsReturned) { + WhiteBoxNumaLibrary::GetMemPolicyPtr memPolicyHandler = + [](int *, unsigned long[], unsigned long, void *, unsigned long) -> long { return 0; }; + WhiteBoxNumaLibrary::NumaAvailablePtr numaAvailableHandler = + [](void) -> int { return 0; }; + WhiteBoxNumaLibrary::NumaMaxNodePtr numaMaxNodeHandler = + [](void) -> int { return 4; }; + MockOsLibrary::loadLibraryNewObject = new MockOsLibraryCustom(nullptr, true); + MockOsLibraryCustom *osLibrary = static_cast(MockOsLibrary::loadLibraryNewObject); + // register proc pointers + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procGetMemPolicyStr)] = reinterpret_cast(memPolicyHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaAvailableStr)] = reinterpret_cast(numaAvailableHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaMaxNodeStr)] = reinterpret_cast(numaMaxNodeHandler); + + WhiteBoxNumaLibrary::osLibraryLoadFunction = MockOsLibraryCustom::load; + EXPECT_TRUE(WhiteBoxNumaLibrary::init()); + EXPECT_TRUE(WhiteBoxNumaLibrary::isLoaded()); + EXPECT_EQ(reinterpret_cast(memPolicyHandler), WhiteBoxNumaLibrary::getMemPolicyFunction); + std::vector memPolicyNodeMask; + int mode = -1; + EXPECT_EQ(true, WhiteBoxNumaLibrary::getMemPolicy(&mode, memPolicyNodeMask, nullptr)); + + MockOsLibrary::loadLibraryNewObject = nullptr; + WhiteBoxNumaLibrary::osLibrary.reset(); +} + +TEST(NumaLibraryTests, givenNumaLibraryWithInvalidMockOsLibraryWhenCallingLoadThenFalseIsReturned) { + WhiteBoxNumaLibrary::GetMemPolicyPtr memPolicyHandler = + [](int *, unsigned long[], unsigned long, void *, unsigned long) -> long { return 0; }; + WhiteBoxNumaLibrary::NumaAvailablePtr numaAvailableHandler = + [](void) -> int { return -1; }; + WhiteBoxNumaLibrary::NumaAvailablePtr numaAvailableHandler2 = + [](void) -> int { return 0; }; + WhiteBoxNumaLibrary::NumaMaxNodePtr numaMaxNodeHandler = + [](void) -> int { return 0; }; + + MockOsLibrary::loadLibraryNewObject = new MockOsLibraryCustom(nullptr, true); + MockOsLibraryCustom *osLibrary = static_cast(MockOsLibrary::loadLibraryNewObject); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaAvailableStr)] = nullptr; + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaMaxNodeStr)] = nullptr; + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procGetMemPolicyStr)] = nullptr; + + WhiteBoxNumaLibrary::osLibraryLoadFunction = MockOsLibraryCustom::load; + EXPECT_FALSE(WhiteBoxNumaLibrary::init()); + EXPECT_FALSE(WhiteBoxNumaLibrary::isLoaded()); + EXPECT_EQ(nullptr, MockOsLibrary::loadLibraryNewObject); + + MockOsLibrary::loadLibraryNewObject = new MockOsLibraryCustom(nullptr, true); + osLibrary = static_cast(MockOsLibrary::loadLibraryNewObject); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaAvailableStr)] = reinterpret_cast(numaAvailableHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaMaxNodeStr)] = nullptr; + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procGetMemPolicyStr)] = nullptr; + EXPECT_FALSE(WhiteBoxNumaLibrary::init()); + EXPECT_FALSE(WhiteBoxNumaLibrary::isLoaded()); + EXPECT_EQ(nullptr, MockOsLibrary::loadLibraryNewObject); + + MockOsLibrary::loadLibraryNewObject = new MockOsLibraryCustom(nullptr, true); + osLibrary = static_cast(MockOsLibrary::loadLibraryNewObject); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaAvailableStr)] = reinterpret_cast(numaAvailableHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaMaxNodeStr)] = reinterpret_cast(numaMaxNodeHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procGetMemPolicyStr)] = nullptr; + EXPECT_FALSE(WhiteBoxNumaLibrary::init()); + EXPECT_FALSE(WhiteBoxNumaLibrary::isLoaded()); + EXPECT_EQ(nullptr, MockOsLibrary::loadLibraryNewObject); + + MockOsLibrary::loadLibraryNewObject = new MockOsLibraryCustom(nullptr, true); + osLibrary = static_cast(MockOsLibrary::loadLibraryNewObject); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaAvailableStr)] = reinterpret_cast(numaAvailableHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaMaxNodeStr)] = reinterpret_cast(numaMaxNodeHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procGetMemPolicyStr)] = reinterpret_cast(memPolicyHandler); + EXPECT_FALSE(WhiteBoxNumaLibrary::init()); + EXPECT_FALSE(WhiteBoxNumaLibrary::isLoaded()); + EXPECT_EQ(nullptr, MockOsLibrary::loadLibraryNewObject); + + MockOsLibrary::loadLibraryNewObject = new MockOsLibraryCustom(nullptr, true); + osLibrary = static_cast(MockOsLibrary::loadLibraryNewObject); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaAvailableStr)] = reinterpret_cast(numaAvailableHandler2); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procNumaMaxNodeStr)] = reinterpret_cast(numaMaxNodeHandler); + osLibrary->procMap[std::string(WhiteBoxNumaLibrary::procGetMemPolicyStr)] = reinterpret_cast(memPolicyHandler); + EXPECT_FALSE(WhiteBoxNumaLibrary::init()); + EXPECT_FALSE(WhiteBoxNumaLibrary::isLoaded()); + EXPECT_EQ(nullptr, MockOsLibrary::loadLibraryNewObject); + + MockOsLibrary::loadLibraryNewObject = nullptr; + EXPECT_FALSE(WhiteBoxNumaLibrary::init()); + EXPECT_FALSE(WhiteBoxNumaLibrary::isLoaded()); + EXPECT_EQ(nullptr, WhiteBoxNumaLibrary::osLibrary.get()); + WhiteBoxNumaLibrary::osLibrary.reset(); +} + +TEST(NumaLibraryTests, givenNumaLibraryWithInvalidOsLibraryWhenCallingGetMemPolicyThenErrorIsReturned) { + MockOsLibrary::loadLibraryNewObject = new MockOsLibrary(nullptr, false); + WhiteBoxNumaLibrary::osLibraryLoadFunction = MockOsLibrary::load; + EXPECT_FALSE(WhiteBoxNumaLibrary::init()); + EXPECT_FALSE(WhiteBoxNumaLibrary::isLoaded()); + + MockOsLibrary::loadLibraryNewObject = nullptr; + WhiteBoxNumaLibrary::osLibrary.reset(); +} + +TEST(NumaLibraryTests, givenNumaLibraryWithInvalidGetMemPolicyWhenCallingGetMemPolicyThenErrorIsReturned) { + MockOsLibrary::loadLibraryNewObject = new MockOsLibrary(nullptr, true); + WhiteBoxNumaLibrary::osLibraryLoadFunction = MockOsLibrary::load; + EXPECT_FALSE(WhiteBoxNumaLibrary::init()); + EXPECT_FALSE(WhiteBoxNumaLibrary::isLoaded()); + EXPECT_EQ(nullptr, WhiteBoxNumaLibrary::getMemPolicyFunction); + std::vector memPolicyNodeMask; + int mode = -1; + EXPECT_EQ(false, WhiteBoxNumaLibrary::getMemPolicy(&mode, memPolicyNodeMask, nullptr)); + MockOsLibrary::loadLibraryNewObject = nullptr; + WhiteBoxNumaLibrary::osLibrary.reset(); +} diff --git a/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp b/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp index fe0a7615e8..e8606034a0 100644 --- a/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -60,7 +60,7 @@ TEST(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingGemCreateExtWithRegionsThen uint32_t numOfChunks = 0; EXPECT_TRUE(xeIoctlHelper->bindInfo.empty()); - EXPECT_NE(0, xeIoctlHelper->createGemExt(memRegions, 0u, handle, 0, {}, -1, false, numOfChunks)); + EXPECT_NE(0, xeIoctlHelper->createGemExt(memRegions, 0u, handle, 0, {}, -1, false, numOfChunks, std::nullopt, std::nullopt)); EXPECT_FALSE(xeIoctlHelper->bindInfo.empty()); EXPECT_EQ(DRM_XE_GEM_CPU_CACHING_WC, drm.createParamsCpuCaching); } @@ -83,7 +83,7 @@ TEST(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingGemCreateExtWithRegionsAndV GemVmControl test = {}; EXPECT_TRUE(xeIoctlHelper->bindInfo.empty()); - EXPECT_NE(0, xeIoctlHelper->createGemExt(memRegions, 0u, handle, 0, test.vmId, -1, false, numOfChunks)); + EXPECT_NE(0, xeIoctlHelper->createGemExt(memRegions, 0u, handle, 0, test.vmId, -1, false, numOfChunks, std::nullopt, std::nullopt)); EXPECT_FALSE(xeIoctlHelper->bindInfo.empty()); EXPECT_EQ(DRM_XE_GEM_CPU_CACHING_WC, drm.createParamsCpuCaching); } @@ -221,7 +221,7 @@ TEST(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingAnyMethodThenDummyValueIsRe MemRegionsVec memRegions{}; uint32_t handle = 0u; uint32_t numOfChunks = 0; - EXPECT_NE(0, xeIoctlHelper->createGemExt(memRegions, 0u, handle, 0, {}, -1, false, numOfChunks)); + EXPECT_NE(0, xeIoctlHelper->createGemExt(memRegions, 0u, handle, 0, {}, -1, false, numOfChunks, std::nullopt, std::nullopt)); EXPECT_TRUE(xeIoctlHelper->isVmBindAvailable()); diff --git a/third_party/numa/LICENSE.LGPL2.1 b/third_party/numa/LICENSE.LGPL2.1 new file mode 100644 index 0000000000..4362b49151 --- /dev/null +++ b/third_party/numa/LICENSE.LGPL2.1 @@ -0,0 +1,502 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/third_party/numa/numaif.h b/third_party/numa/numaif.h new file mode 100644 index 0000000000..91aa230aed --- /dev/null +++ b/third_party/numa/numaif.h @@ -0,0 +1,48 @@ +#ifndef NUMAIF_H +#define NUMAIF_H 1 + +#ifdef __cplusplus +extern "C" { +#endif + +/* Kernel interface for NUMA API */ + +/* System calls */ +extern long get_mempolicy(int *mode, unsigned long *nmask, + unsigned long maxnode, void *addr, unsigned flags); +extern long mbind(void *start, unsigned long len, int mode, + const unsigned long *nmask, unsigned long maxnode, unsigned flags); +extern long set_mempolicy(int mode, const unsigned long *nmask, + unsigned long maxnode); +extern long migrate_pages(int pid, unsigned long maxnode, + const unsigned long *frommask, + const unsigned long *tomask); + +extern long move_pages(int pid, unsigned long count, + void **pages, const int *nodes, int *status, int flags); + +/* Policies */ +#define MPOL_DEFAULT 0 +#define MPOL_PREFERRED 1 +#define MPOL_BIND 2 +#define MPOL_INTERLEAVE 3 +#define MPOL_LOCAL 4 +#define MPOL_MAX 5 + +/* Flags for get_mem_policy */ +#define MPOL_F_NODE (1<<0) /* return next il node or node of address */ + /* Warning: MPOL_F_NODE is unsupported and + subject to change. Don't use. */ +#define MPOL_F_ADDR (1<<1) /* look up vma using address */ +#define MPOL_F_MEMS_ALLOWED (1<<2) /* query nodes allowed in cpuset */ + +/* Flags for mbind */ +#define MPOL_MF_STRICT (1<<0) /* Verify existing pages in the mapping */ +#define MPOL_MF_MOVE (1<<1) /* Move pages owned by this process to conform to mapping */ +#define MPOL_MF_MOVE_ALL (1<<2) /* Move every page to conform to mapping */ + +#ifdef __cplusplus +} +#endif + +#endif