fix: return correct memory values for integrated platforms
Related-To: NEO-14198 Signed-off-by: Shreyas Kunder <shreyas.kunder@intel.com>
This commit is contained in:
parent
8182671124
commit
766da0f4ad
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2020-2024 Intel Corporation
|
||||
* Copyright (C) 2020-2025 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
|
@ -17,6 +17,7 @@
|
|||
#include "level_zero/sysman/source/shared/firmware_util/sysman_firmware_util.h"
|
||||
#include "level_zero/sysman/source/shared/linux/kmd_interface/sysman_kmd_interface.h"
|
||||
#include "level_zero/sysman/source/shared/linux/product_helper/sysman_product_helper.h"
|
||||
#include "level_zero/sysman/source/shared/linux/sysman_fs_access_interface.h"
|
||||
#include "level_zero/sysman/source/shared/linux/zes_os_sysman_imp.h"
|
||||
|
||||
namespace L0 {
|
||||
|
@ -53,6 +54,20 @@ ze_result_t LinuxMemoryImp::getState(zes_mem_state_t *pState) {
|
|||
"Error@ %s():createMemoryInfo failed errno:%d \n", __FUNCTION__, errno);
|
||||
return status;
|
||||
}
|
||||
if (pLinuxSysmanImp->getHardwareInfo().capabilityTable.isIntegratedDevice) {
|
||||
const std::string memFreeKey = "MemFree";
|
||||
const std::string memAvailableKey = "MemAvailable";
|
||||
auto memInfoValues = readMemInfoValues(&pLinuxSysmanImp->getFsAccess(), {memFreeKey, memAvailableKey});
|
||||
if (memInfoValues.find(memFreeKey) != memInfoValues.end() && memInfoValues.find(memAvailableKey) != memInfoValues.end()) {
|
||||
pState->free = memInfoValues[memFreeKey] * 1024;
|
||||
pState->size = memInfoValues[memAvailableKey] * 1024;
|
||||
} else {
|
||||
pState->free = 0;
|
||||
pState->size = 0;
|
||||
status = ZE_RESULT_ERROR_UNKNOWN;
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
auto region = memoryInfo->getMemoryRegion(MemoryBanks::getBankForLocalMemory(subdeviceId));
|
||||
pState->free = region.unallocatedSize;
|
||||
|
@ -60,6 +75,35 @@ ze_result_t LinuxMemoryImp::getState(zes_mem_state_t *pState) {
|
|||
return status;
|
||||
}
|
||||
|
||||
std::map<std::string, uint64_t> LinuxMemoryImp::readMemInfoValues(FsAccessInterface *pFsAccess, const std::vector<std::string> &keys) {
|
||||
std::map<std::string, uint64_t> result = {};
|
||||
const std::string memInfoFile = "/proc/meminfo";
|
||||
std::vector<std::string> memInfo{};
|
||||
|
||||
if (pFsAccess->read(memInfoFile, memInfo) == ZE_RESULT_SUCCESS) {
|
||||
for (const auto &line : memInfo) {
|
||||
std::istringstream lineStream(line);
|
||||
std::string label = "";
|
||||
std::string unit = "";
|
||||
uint64_t value = 0;
|
||||
|
||||
lineStream >> label >> value >> unit;
|
||||
|
||||
if (!label.empty() && label.back() == ':') {
|
||||
label.pop_back();
|
||||
}
|
||||
|
||||
if (std::find(keys.begin(), keys.end(), label) != keys.end()) {
|
||||
result[label] = value;
|
||||
if (result.size() == keys.size()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
LinuxMemoryImp::LinuxMemoryImp(OsSysman *pOsSysman, ze_bool_t onSubdevice, uint32_t subdeviceId) : isSubdevice(onSubdevice), subdeviceId(subdeviceId) {
|
||||
pLinuxSysmanImp = static_cast<LinuxSysmanImp *>(pOsSysman);
|
||||
pDrm = pLinuxSysmanImp->getDrm();
|
||||
|
@ -69,6 +113,9 @@ LinuxMemoryImp::LinuxMemoryImp(OsSysman *pOsSysman, ze_bool_t onSubdevice, uint3
|
|||
|
||||
bool LinuxMemoryImp::isMemoryModuleSupported() {
|
||||
auto &gfxCoreHelper = pDevice->getRootDeviceEnvironment().getHelper<NEO::GfxCoreHelper>();
|
||||
if (pLinuxSysmanImp->getHardwareInfo().capabilityTable.isIntegratedDevice) {
|
||||
return true;
|
||||
}
|
||||
return gfxCoreHelper.getEnableLocalMemory(pDevice->getHardwareInfo());
|
||||
}
|
||||
|
||||
|
|
|
@ -10,7 +10,9 @@
|
|||
|
||||
#include "level_zero/sysman/source/api/memory/sysman_os_memory.h"
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace NEO {
|
||||
class Drm;
|
||||
|
@ -22,12 +24,14 @@ namespace Sysman {
|
|||
class LinuxSysmanImp;
|
||||
class SysmanKmdInterface;
|
||||
struct SysmanDeviceImp;
|
||||
class FsAccessInterface;
|
||||
|
||||
class LinuxMemoryImp : public OsMemory, NEO::NonCopyableAndNonMovableClass {
|
||||
public:
|
||||
ze_result_t getProperties(zes_mem_properties_t *pProperties) override;
|
||||
ze_result_t getBandwidth(zes_mem_bandwidth_t *pBandwidth) override;
|
||||
ze_result_t getState(zes_mem_state_t *pState) override;
|
||||
static std::map<std::string, uint64_t> readMemInfoValues(FsAccessInterface *pFsAccess, const std::vector<std::string> &keys);
|
||||
bool isMemoryModuleSupported() override;
|
||||
LinuxMemoryImp(OsSysman *pOsSysman, ze_bool_t onSubdevice, uint32_t subdeviceId);
|
||||
LinuxMemoryImp() = default;
|
||||
|
|
|
@ -9,6 +9,7 @@
|
|||
#include "shared/source/os_interface/linux/memory_info.h"
|
||||
#include "shared/source/os_interface/linux/system_info.h"
|
||||
|
||||
#include "level_zero/sysman/source/api/memory/linux/sysman_os_memory_imp.h"
|
||||
#include "level_zero/sysman/source/api/ras/linux/ras_util/sysman_ras_util.h"
|
||||
#include "level_zero/sysman/source/shared/firmware_util/sysman_firmware_util.h"
|
||||
#include "level_zero/sysman/source/shared/linux/kmd_interface/sysman_kmd_interface.h"
|
||||
|
@ -37,8 +38,13 @@ void SysmanProductHelperHw<gfxProduct>::getFrequencyStepSize(double *pStepSize)
|
|||
template <PRODUCT_FAMILY gfxProduct>
|
||||
ze_result_t SysmanProductHelperHw<gfxProduct>::getMemoryProperties(zes_mem_properties_t *pProperties, LinuxSysmanImp *pLinuxSysmanImp, NEO::Drm *pDrm, SysmanKmdInterface *pSysmanKmdInterface, uint32_t subDeviceId, bool isSubdevice) {
|
||||
auto pSysFsAccess = pSysmanKmdInterface->getSysFsAccess();
|
||||
bool isIntegratedDevice = pLinuxSysmanImp->getHardwareInfo().capabilityTable.isIntegratedDevice;
|
||||
|
||||
pProperties->location = ZES_MEM_LOC_DEVICE;
|
||||
if (isIntegratedDevice) {
|
||||
pProperties->location = ZES_MEM_LOC_SYSTEM;
|
||||
} else {
|
||||
pProperties->location = ZES_MEM_LOC_DEVICE;
|
||||
}
|
||||
pProperties->type = ZES_MEM_TYPE_DDR;
|
||||
pProperties->onSubdevice = isSubdevice;
|
||||
pProperties->subdeviceId = subDeviceId;
|
||||
|
@ -84,7 +90,17 @@ ze_result_t SysmanProductHelperHw<gfxProduct>::getMemoryProperties(zes_mem_prope
|
|||
pProperties->busWidth = memoryBusWidth;
|
||||
pProperties->physicalSize = 0;
|
||||
|
||||
if (pSysmanKmdInterface->isPhysicalMemorySizeSupported() == true) {
|
||||
if (isIntegratedDevice) {
|
||||
pProperties->busWidth = -1;
|
||||
pProperties->numChannels = -1;
|
||||
pProperties->type = ZES_MEM_TYPE_FORCE_UINT32;
|
||||
|
||||
const std::string memTotalKey = "MemTotal";
|
||||
auto memInfoValues = LinuxMemoryImp::readMemInfoValues(&pLinuxSysmanImp->getFsAccess(), {memTotalKey});
|
||||
if (memInfoValues.find(memTotalKey) != memInfoValues.end()) {
|
||||
pProperties->physicalSize = memInfoValues[memTotalKey] * 1024;
|
||||
}
|
||||
} else if (pSysmanKmdInterface->isPhysicalMemorySizeSupported() == true) {
|
||||
if (isSubdevice) {
|
||||
std::string memval;
|
||||
std::string physicalSizeFile = pSysmanKmdInterface->getSysfsFilePathForPhysicalMemorySize(subDeviceId);
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2020-2024 Intel Corporation
|
||||
* Copyright (C) 2020-2025 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
|
@ -68,6 +68,10 @@ constexpr uint64_t mockIdiWriteVal = 9u;
|
|||
constexpr uint64_t mockDisplayVc1ReadVal = 10u;
|
||||
constexpr uint64_t numberMcChannels = 16;
|
||||
|
||||
constexpr uint64_t mockIntegratedDeviceAvailableMemory = 8192 * 1024;
|
||||
constexpr uint64_t mockIntegratedDeviceFreeMemory = 4096 * 1024;
|
||||
constexpr uint64_t mockIntegratedDevicePhysicalSize = 16384 * 1024;
|
||||
|
||||
namespace L0 {
|
||||
namespace Sysman {
|
||||
namespace ult {
|
||||
|
@ -167,6 +171,26 @@ struct MockMemorySysFsAccessInterface : public L0::Sysman::SysFsAccessInterface
|
|||
}
|
||||
};
|
||||
|
||||
struct MockMemoryFsAccessInterface : public L0::Sysman::FsAccessInterface {
|
||||
bool mockMemInfoIncorrectValue = false;
|
||||
ze_result_t read(std::string file, std::vector<std::string> &val) override {
|
||||
if (file == "/proc/meminfo") {
|
||||
if (mockMemInfoIncorrectValue) {
|
||||
val.push_back("Buffers: 158772 kB");
|
||||
val.push_back("Cached: 11744244 kB");
|
||||
val.push_back("SwapCached: 1376 kB");
|
||||
val.push_back("Active: 6777644 kB");
|
||||
} else {
|
||||
val.push_back("MemTotal: 16384 kB");
|
||||
val.push_back("MemFree: 4096 kB");
|
||||
val.push_back("MemAvailable: 8192 kB");
|
||||
}
|
||||
}
|
||||
return ZE_RESULT_SUCCESS;
|
||||
}
|
||||
MockMemoryFsAccessInterface() = default;
|
||||
};
|
||||
|
||||
class MockProcFsAccessInterface : public L0::Sysman::ProcFsAccessInterface {
|
||||
public:
|
||||
MockProcFsAccessInterface() = default;
|
||||
|
|
|
@ -202,22 +202,38 @@ TEST_F(SysmanDeviceMemoryFixtureI915, GivenI915DriverVersionWhenValidCallingSysf
|
|||
EXPECT_STREQ("gt/gt0/mem_RPn_freq_mhz", pSysmanKmdInterface->getSysfsFilePath(SysfsName::sysfsNameMinMemoryFrequency, 0, true).c_str());
|
||||
}
|
||||
|
||||
TEST_F(SysmanDeviceMemoryFixtureI915, GivenComponentCountZeroWhenEnumeratingMemoryModulesWithNoLocalMemorySupportThenZeroCountIsReturned) {
|
||||
setLocalSupportedAndReinit(false);
|
||||
TEST_F(SysmanDeviceMemoryFixtureI915, GivenComponentCountZeroWhenEnumeratingMemoryModulesWithNoLocalMemorySupportThenZeroCountIsReturnedForDiscretePlatforms) {
|
||||
if (!defaultHwInfo->capabilityTable.isIntegratedDevice) {
|
||||
setLocalSupportedAndReinit(false);
|
||||
}
|
||||
uint32_t count = 0;
|
||||
EXPECT_EQ(zesDeviceEnumMemoryModules(device->toHandle(), &count, nullptr), ZE_RESULT_SUCCESS);
|
||||
EXPECT_EQ(count, 0u);
|
||||
if (defaultHwInfo->capabilityTable.isIntegratedDevice) {
|
||||
EXPECT_EQ(count, 1u);
|
||||
} else {
|
||||
EXPECT_EQ(count, 0u);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(SysmanDeviceMemoryFixtureI915, GivenInvalidComponentCountWhenEnumeratingMemoryModulesWithNoLocalMemorySupportThenZeroCountIsReturned) {
|
||||
setLocalSupportedAndReinit(false);
|
||||
TEST_F(SysmanDeviceMemoryFixtureI915, GivenInvalidComponentCountWhenEnumeratingMemoryModulesWithNoLocalMemorySupportThenZeroCountIsReturnedForDiscretePlatforms) {
|
||||
if (!defaultHwInfo->capabilityTable.isIntegratedDevice) {
|
||||
setLocalSupportedAndReinit(false);
|
||||
}
|
||||
uint32_t count = 0;
|
||||
EXPECT_EQ(zesDeviceEnumMemoryModules(device->toHandle(), &count, nullptr), ZE_RESULT_SUCCESS);
|
||||
EXPECT_EQ(count, 0u);
|
||||
if (defaultHwInfo->capabilityTable.isIntegratedDevice) {
|
||||
EXPECT_EQ(count, 1u);
|
||||
} else {
|
||||
EXPECT_EQ(count, 0u);
|
||||
}
|
||||
|
||||
count = count + 1;
|
||||
EXPECT_EQ(zesDeviceEnumMemoryModules(device->toHandle(), &count, nullptr), ZE_RESULT_SUCCESS);
|
||||
EXPECT_EQ(count, 0u);
|
||||
if (defaultHwInfo->capabilityTable.isIntegratedDevice) {
|
||||
EXPECT_EQ(count, 1u);
|
||||
} else {
|
||||
EXPECT_EQ(count, 0u);
|
||||
}
|
||||
}
|
||||
|
||||
HWTEST2_F(SysmanDeviceMemoryFixtureI915, GivenComponentCountZeroWhenEnumeratingMemoryModulesThenValidCountIsReturned, IsPVC) {
|
||||
|
@ -386,7 +402,11 @@ TEST_F(SysmanDeviceMemoryFixtureI915, GivenValidMemoryHandleWhenCallingZesMemory
|
|||
ze_result_t result = zesMemoryGetProperties(handle, &properties);
|
||||
EXPECT_EQ(result, ZE_RESULT_SUCCESS);
|
||||
EXPECT_EQ(properties.type, ZES_MEM_TYPE_DDR);
|
||||
EXPECT_EQ(properties.location, ZES_MEM_LOC_DEVICE);
|
||||
if (defaultHwInfo->capabilityTable.isIntegratedDevice) {
|
||||
EXPECT_EQ(properties.location, ZES_MEM_LOC_SYSTEM);
|
||||
} else {
|
||||
EXPECT_EQ(properties.location, ZES_MEM_LOC_DEVICE);
|
||||
}
|
||||
EXPECT_EQ(properties.numChannels, -1);
|
||||
EXPECT_FALSE(properties.onSubdevice);
|
||||
EXPECT_EQ(properties.subdeviceId, 0u);
|
||||
|
@ -795,6 +815,7 @@ class SysmanMultiDeviceMemoryFixture : public SysmanMultiDeviceFixture {
|
|||
L0::Sysman::SysmanDevice *device = nullptr;
|
||||
MockMemorySysFsAccessInterface *pSysfsAccess = nullptr;
|
||||
MockSysmanKmdInterfacePrelim *pSysmanKmdInterface = nullptr;
|
||||
MockMemoryFsAccessInterface *pFsAccess = nullptr;
|
||||
|
||||
void SetUp() override {
|
||||
debugManager.flags.EnableLocalMemory.set(1);
|
||||
|
@ -802,9 +823,12 @@ class SysmanMultiDeviceMemoryFixture : public SysmanMultiDeviceFixture {
|
|||
|
||||
pSysmanKmdInterface = new MockSysmanKmdInterfacePrelim(pLinuxSysmanImp->getSysmanProductHelper());
|
||||
pSysfsAccess = new MockMemorySysFsAccessInterface();
|
||||
pFsAccess = new MockMemoryFsAccessInterface();
|
||||
pSysmanKmdInterface->pSysfsAccess.reset(pSysfsAccess);
|
||||
pSysmanKmdInterface->pFsAccess.reset(pFsAccess);
|
||||
pLinuxSysmanImp->pSysmanKmdInterface.reset(pSysmanKmdInterface);
|
||||
pLinuxSysmanImp->pSysfsAccess = pLinuxSysmanImp->pSysmanKmdInterface->getSysFsAccess();
|
||||
pLinuxSysmanImp->pFsAccess = pLinuxSysmanImp->pSysmanKmdInterface->getFsAccess();
|
||||
pDrm = new MockMemoryNeoDrm(const_cast<NEO::RootDeviceEnvironment &>(pSysmanDeviceImp->getRootDeviceEnvironment()));
|
||||
pDrm->ioctlHelper = static_cast<std::unique_ptr<NEO::IoctlHelper>>(std::make_unique<NEO::MockIoctlHelper>(*pDrm));
|
||||
auto &osInterface = pSysmanDeviceImp->getRootDeviceEnvironment().osInterface;
|
||||
|
@ -866,15 +890,36 @@ HWTEST2_F(SysmanMultiDeviceMemoryFixture, GivenValidMemoryHandleWhenCallingZesMe
|
|||
ze_result_t result = zesMemoryGetState(handles[0], &state1);
|
||||
EXPECT_EQ(result, ZE_RESULT_SUCCESS);
|
||||
EXPECT_EQ(state1.health, ZES_MEM_HEALTH_UNKNOWN);
|
||||
EXPECT_EQ(state1.size, NEO::probedSizeRegionOne);
|
||||
EXPECT_EQ(state1.free, NEO::unallocatedSizeRegionOne);
|
||||
if (defaultHwInfo->capabilityTable.isIntegratedDevice) {
|
||||
EXPECT_EQ(state1.size, mockIntegratedDeviceAvailableMemory);
|
||||
EXPECT_EQ(state1.free, mockIntegratedDeviceFreeMemory);
|
||||
} else {
|
||||
EXPECT_EQ(state1.size, NEO::probedSizeRegionOne);
|
||||
EXPECT_EQ(state1.free, NEO::unallocatedSizeRegionOne);
|
||||
}
|
||||
|
||||
zes_mem_state_t state2;
|
||||
result = zesMemoryGetState(handles[1], &state2);
|
||||
EXPECT_EQ(result, ZE_RESULT_SUCCESS);
|
||||
EXPECT_EQ(state2.health, ZES_MEM_HEALTH_UNKNOWN);
|
||||
EXPECT_EQ(state2.size, NEO::probedSizeRegionFour);
|
||||
EXPECT_EQ(state2.free, NEO::unallocatedSizeRegionFour);
|
||||
if (defaultHwInfo->capabilityTable.isIntegratedDevice) {
|
||||
EXPECT_EQ(state2.size, mockIntegratedDeviceAvailableMemory);
|
||||
EXPECT_EQ(state2.free, mockIntegratedDeviceFreeMemory);
|
||||
} else {
|
||||
EXPECT_EQ(state2.size, NEO::probedSizeRegionFour);
|
||||
EXPECT_EQ(state2.free, NEO::unallocatedSizeRegionFour);
|
||||
}
|
||||
}
|
||||
|
||||
HWTEST2_F(SysmanMultiDeviceMemoryFixture, GivenMemFreeAndMemAvailableMissingInMemInfoWhenCallingGetStateThenFreeAndSizeValuesAreZero, IsMTL) {
|
||||
pFsAccess->mockMemInfoIncorrectValue = true;
|
||||
auto handles = getMemoryHandles(pOsSysman->getSubDeviceCount());
|
||||
zes_mem_state_t state = {};
|
||||
ze_result_t result = zesMemoryGetState(handles[0], &state);
|
||||
|
||||
EXPECT_EQ(result, ZE_RESULT_ERROR_UNKNOWN);
|
||||
EXPECT_EQ(state.free, 0u);
|
||||
EXPECT_EQ(state.size, 0u);
|
||||
}
|
||||
|
||||
} // namespace ult
|
||||
|
|
|
@ -921,20 +921,31 @@ HWTEST2_F(SysmanProductHelperMemoryTest, GivenSysmanProductHelperInstanceWhenCal
|
|||
|
||||
auto pSysmanKmdInterface = new MockSysmanKmdInterfacePrelim(pLinuxSysmanImp->getSysmanProductHelper());
|
||||
MockMemorySysFsAccessInterface *pSysfsAccess = new MockMemorySysFsAccessInterface();
|
||||
MockMemoryFsAccessInterface *pFsAccess = new MockMemoryFsAccessInterface();
|
||||
pLinuxSysmanImp->pSysmanKmdInterface.reset(pSysmanKmdInterface);
|
||||
pSysmanKmdInterface->pSysfsAccess.reset(pSysfsAccess);
|
||||
pSysmanKmdInterface->pFsAccess.reset(pFsAccess);
|
||||
pLinuxSysmanImp->pFsAccess = pLinuxSysmanImp->pSysmanKmdInterface->getFsAccess();
|
||||
pSysfsAccess->mockReadStringValue.push_back(mockPhysicalSize);
|
||||
pSysfsAccess->mockReadReturnStatus.push_back(ZE_RESULT_SUCCESS);
|
||||
pSysfsAccess->isRepeated = true;
|
||||
|
||||
ze_result_t result = pSysmanProductHelper->getMemoryProperties(&properties, pLinuxSysmanImp, pDrm.get(), pLinuxSysmanImp->getSysmanKmdInterface(), subDeviceId, isSubdevice);
|
||||
EXPECT_EQ(result, ZE_RESULT_SUCCESS);
|
||||
EXPECT_EQ(properties.type, ZES_MEM_TYPE_HBM);
|
||||
EXPECT_EQ(properties.location, ZES_MEM_LOC_DEVICE);
|
||||
EXPECT_EQ(properties.subdeviceId, 0u);
|
||||
EXPECT_EQ(properties.physicalSize, strtoull(mockPhysicalSize.c_str(), nullptr, 16));
|
||||
EXPECT_EQ(properties.numChannels, numMemoryChannels);
|
||||
EXPECT_EQ(properties.busWidth, memoryBusWidth);
|
||||
if (defaultHwInfo->capabilityTable.isIntegratedDevice) {
|
||||
EXPECT_EQ(properties.location, ZES_MEM_LOC_SYSTEM);
|
||||
EXPECT_EQ(properties.numChannels, -1);
|
||||
EXPECT_EQ(properties.busWidth, -1);
|
||||
EXPECT_EQ(properties.type, ZES_MEM_TYPE_FORCE_UINT32);
|
||||
EXPECT_EQ(properties.physicalSize, mockIntegratedDevicePhysicalSize);
|
||||
} else {
|
||||
EXPECT_EQ(properties.location, ZES_MEM_LOC_DEVICE);
|
||||
EXPECT_EQ(properties.numChannels, numMemoryChannels);
|
||||
EXPECT_EQ(properties.busWidth, memoryBusWidth);
|
||||
EXPECT_EQ(properties.type, ZES_MEM_TYPE_HBM);
|
||||
EXPECT_EQ(properties.subdeviceId, 0u);
|
||||
EXPECT_EQ(properties.physicalSize, strtoull(mockPhysicalSize.c_str(), nullptr, 16));
|
||||
}
|
||||
}
|
||||
|
||||
HWTEST2_F(SysmanProductHelperMemoryTest, GivenSysmanProductHelperInstanceWhenCallingGetMemoryBandwidthAndNoTelemNodesAvailableThenFailureIsReturned, IsBMG) {
|
||||
|
|
Loading…
Reference in New Issue