Sysman: Add support to check if resizable Bar supported

Signed-off-by: Jitendra Sharma <jitendra.sharma@intel.com>
This commit is contained in:
Jitendra Sharma
2021-07-12 06:23:08 +00:00
committed by Compute-Runtime-Automation
parent c96c1b045a
commit 478edfc0e4
9 changed files with 275 additions and 11 deletions

View File

@@ -577,4 +577,8 @@ bool SysfsAccess::isMyDeviceFile(const std::string dev) {
return false;
}
bool SysfsAccess::isRootUser() {
return FsAccess::isRootUser();
}
} // namespace L0

View File

@@ -111,6 +111,7 @@ class SysfsAccess : protected FsAccess {
MOCKABLE_VIRTUAL bool fileExists(const std::string file) override;
MOCKABLE_VIRTUAL bool isMyDeviceFile(const std::string dev);
MOCKABLE_VIRTUAL bool directoryExists(const std::string path) override;
MOCKABLE_VIRTUAL bool isRootUser() override;
private:
SysfsAccess(const std::string file);

View File

@@ -12,6 +12,8 @@
#include "sysman/pci/pci_imp.h"
#include <linux/pci_regs.h>
namespace L0 {
const std::string LinuxPciImp::deviceDir("device");
@@ -150,7 +152,34 @@ ze_result_t LinuxPciImp::initializeBarProperties(std::vector<zes_pci_bar_propert
return result;
}
// Parse PCIe configuration space to see if resizable Bar is supported
bool LinuxPciImp::resizableBarSupported() {
uint32_t pos = PCI_CFG_SPACE_SIZE;
uint32_t header = 0;
if (!configMemory) {
return false;
}
// Minimum 8 bytes per capability. Hence maximum capabilities that
// could be present in PCI extended configuration space are
// represented by loopCount.
auto loopCount = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
header = getDwordFromConfig(pos);
if (!header) {
return false;
}
while (loopCount-- > 0) {
if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_REBAR) {
return true;
}
pos = PCI_EXT_CAP_NEXT(header);
if (pos < PCI_CFG_SPACE_SIZE) {
return false;
}
header = getDwordFromConfig(pos);
}
return false;
}
@@ -161,12 +190,30 @@ bool LinuxPciImp::resizableBarEnabled() {
ze_result_t LinuxPciImp::getState(zes_pci_state_t *state) {
return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
}
void LinuxPciImp::pciExtendedConfigRead() {
std::string pciConfigNode;
pSysfsAccess->getRealPath("device/config", pciConfigNode);
int fdConfig = -1;
fdConfig = this->openFunction(pciConfigNode.c_str(), O_RDONLY);
if (fdConfig < 0) {
return;
}
configMemory = std::make_unique<uint8_t[]>(PCI_CFG_SPACE_EXP_SIZE);
memset(configMemory.get(), 0, PCI_CFG_SPACE_EXP_SIZE);
this->preadFunction(fdConfig, configMemory.get(), PCI_CFG_SPACE_EXP_SIZE, 0);
this->closeFunction(fdConfig);
}
LinuxPciImp::LinuxPciImp(OsSysman *pOsSysman) {
pLinuxSysmanImp = static_cast<LinuxSysmanImp *>(pOsSysman);
pSysfsAccess = &pLinuxSysmanImp->getSysfsAccess();
pfsAccess = &pLinuxSysmanImp->getFsAccess();
Device *pDevice = pLinuxSysmanImp->getDeviceHandle();
isLmemSupported = pDevice->getDriverHandle()->getMemoryManager()->isLocalMemorySupported(pDevice->getRootDeviceIndex());
if (pSysfsAccess->isRootUser()) {
pciExtendedConfigRead();
}
}
OsPci *OsPci::create(OsSysman *pOsSysman) {

View File

@@ -7,6 +7,7 @@
#pragma once
#include "shared/source/helpers/non_copyable_or_moveable.h"
#include "shared/source/os_interface/linux/sys_calls.h"
#include "sysman/linux/os_sysman_imp.h"
#include "sysman/pci/os_pci.h"
@@ -34,6 +35,11 @@ class LinuxPciImp : public OsPci, NEO::NonCopyableOrMovableClass {
SysfsAccess *pSysfsAccess = nullptr;
FsAccess *pfsAccess = nullptr;
LinuxSysmanImp *pLinuxSysmanImp = nullptr;
std::unique_ptr<uint8_t[]> configMemory;
void pciExtendedConfigRead();
decltype(&NEO::SysCalls::open) openFunction = NEO::SysCalls::open;
decltype(&NEO::SysCalls::close) closeFunction = NEO::SysCalls::close;
decltype(&NEO::SysCalls::pread) preadFunction = NEO::SysCalls::pread;
private:
static const std::string deviceDir;
@@ -41,6 +47,10 @@ class LinuxPciImp : public OsPci, NEO::NonCopyableOrMovableClass {
static const std::string maxLinkSpeedFile;
static const std::string maxLinkWidthFile;
bool isLmemSupported = false;
uint32_t getDwordFromConfig(uint32_t pos) {
return configMemory[pos] | (configMemory[pos + 1] << 8) |
(configMemory[pos + 2] << 16) | (configMemory[pos + 3] << 24);
}
};
} // namespace L0

View File

@@ -106,8 +106,8 @@ ze_result_t PciImp::pciGetInitializedBars(uint32_t *pCount, zes_pci_bar_properti
pBarPropsExt->index = pciBarProperties[i]->index;
pBarPropsExt->size = pciBarProperties[i]->size;
pBarPropsExt->type = pciBarProperties[i]->type;
pBarPropsExt->resizableBarEnabled = resizableBarEnabled;
pBarPropsExt->resizableBarSupported = resizableBarSupported;
pBarPropsExt->resizableBarEnabled = static_cast<ze_bool_t>(resizableBarEnabled);
pBarPropsExt->resizableBarSupported = static_cast<ze_bool_t>(resizableBarSupported);
}
}
}
@@ -117,11 +117,8 @@ ze_result_t PciImp::pciGetInitializedBars(uint32_t *pCount, zes_pci_bar_properti
ze_result_t PciImp::pciGetState(zes_pci_state_t *pState) {
return pOsPci->getState(pState);
}
void PciImp::init() {
if (pOsPci == nullptr) {
pOsPci = OsPci::create(pOsSysman);
}
UNRECOVERABLE_IF(nullptr == pOsPci);
void PciImp::pciGetStaticFields() {
pOsPci->getProperties(&pciProperties);
resizableBarEnabled = pOsPci->resizableBarEnabled();
resizableBarSupported = pOsPci->resizableBarSupported();
@@ -154,6 +151,15 @@ void PciImp::init() {
pOsPci->initializeBarProperties(pciBarProperties);
}
void PciImp::init() {
if (pOsPci == nullptr) {
pOsPci = OsPci::create(pOsSysman);
}
UNRECOVERABLE_IF(nullptr == pOsPci);
pciGetStaticFields();
}
PciImp::~PciImp() {
for (zes_pci_bar_properties_t *pProperties : pciBarProperties) {
delete pProperties;

View File

@@ -25,6 +25,7 @@ class PciImp : public Pci, NEO::NonCopyableOrMovableClass {
ze_result_t pciStaticProperties(zes_pci_properties_t *pProperties) override;
ze_result_t pciGetInitializedBars(uint32_t *pCount, zes_pci_bar_properties_t *pProperties) override;
ze_result_t pciGetState(zes_pci_state_t *pState) override;
void pciGetStaticFields();
PciImp() = default;
PciImp(OsSysman *pOsSysman) : pOsSysman(pOsSysman){};

View File

@@ -302,6 +302,12 @@ void testSysmanPci(ze_device_handle_t &device) {
std::cout << "Bar count = " << count << std::endl;
}
std::vector<zes_pci_bar_properties_t> pciBarProps(count);
zes_pci_bar_properties_1_2_t props1_2 = {};
memset(&props1_2, 0, sizeof(zes_pci_bar_properties_1_2_t));
for (uint32_t i = 0; i < count; i++) {
pciBarProps[i].pNext = static_cast<void *>(&props1_2);
pciBarProps[i].stype = ZES_STRUCTURE_TYPE_PCI_BAR_PROPERTIES_1_2;
}
VALIDATECALL(zesDevicePciGetBars(device, &count, pciBarProps.data()));
if (verbose) {
for (uint32_t i = 0; i < count; i++) {
@@ -309,6 +315,8 @@ void testSysmanPci(ze_device_handle_t &device) {
std::cout << "pciBarProps.index = " << std::hex << pciBarProps[i].index << std::endl;
std::cout << "pciBarProps.base = " << std::hex << pciBarProps[i].base << std::endl;
std::cout << "pciBarProps.size = " << std::hex << pciBarProps[i].size << std::endl;
std::cout << "pci_bar_properties_1_2_t.resizableBarSupported = " << static_cast<uint32_t>(props1_2.resizableBarSupported) << std::endl;
std::cout << "pci_bar_properties_1_2_t.resizableBarEnabled = " << static_cast<uint32_t>(props1_2.resizableBarEnabled) << std::endl;
}
}
}

View File

@@ -21,6 +21,7 @@ const std::string maxLinkSpeedFile("device/max_link_speed");
const std::string maxLinkWidthFile("device/max_link_width");
const std::string mockBdf = "0000:00:02.0";
const std::string mockRealPath = "/sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0/0000:02:01.0/" + mockBdf;
const std::string mockRealPathConfig = mockRealPath + "/config";
const std::string mockRealPath2LevelsUp = "/sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0";
constexpr double mockMaxLinkSpeed = 2.5;
@@ -83,6 +84,11 @@ struct Mock<PciSysfsAccess> : public PciSysfsAccess {
MOCK_METHOD(ze_result_t, read, (const std::string file, std::vector<std::string> &val), (override));
MOCK_METHOD(ze_result_t, readSymLink, (const std::string file, std::string &buf), (override));
MOCK_METHOD(ze_result_t, getRealPath, (const std::string file, std::string &buf), (override));
MOCK_METHOD(bool, isRootUser, (), (override));
bool checkRootUser() {
return true;
}
ze_result_t getValDouble(const std::string file, double &val) {
if (file.compare(maxLinkSpeedFile) == 0) {
@@ -120,6 +126,10 @@ struct Mock<PciSysfsAccess> : public PciSysfsAccess {
val = mockRealPath;
return ZE_RESULT_SUCCESS;
}
if (file.compare("device/config") == 0) {
val = mockRealPathConfig;
return ZE_RESULT_SUCCESS;
}
return ZE_RESULT_ERROR_NOT_AVAILABLE;
}
@@ -136,7 +146,13 @@ struct Mock<PciSysfsAccess> : public PciSysfsAccess {
class PublicLinuxPciImp : public L0::LinuxPciImp {
public:
PublicLinuxPciImp(OsSysman *pOsSysman) : LinuxPciImp(pOsSysman) {}
using LinuxPciImp::closeFunction;
using LinuxPciImp::configMemory;
using LinuxPciImp::openFunction;
using LinuxPciImp::pciExtendedConfigRead;
using LinuxPciImp::pfsAccess;
using LinuxPciImp::preadFunction;
using LinuxPciImp::pSysfsAccess;
};

View File

@@ -32,6 +32,93 @@ constexpr int convertMegabitsPerSecondToBytesPerSecond = 125000;
constexpr int convertGigabitToMegabit = 1000;
constexpr double encodingGen1Gen2 = 0.8;
constexpr double encodingGen3andAbove = 0.98461538461;
constexpr int pciExtendedConfigSpaceSize = 4096;
static int fakeFileDescriptor = 123;
inline static int openMock(const char *pathname, int flags) {
if (strcmp(pathname, mockRealPathConfig.c_str()) == 0) {
return fakeFileDescriptor;
}
return -1;
}
inline static int openMockReturnFailure(const char *pathname, int flags) {
return -1;
}
inline static int closeMock(int fd) {
if (fd == fakeFileDescriptor) {
return 0;
}
return -1;
}
ssize_t preadMock(int fd, void *buf, size_t count, off_t offset) {
uint8_t *mockBuf = static_cast<uint8_t *>(buf);
// Sample config values
mockBuf[0x100] = 0x0e;
mockBuf[0x102] = 0x01;
mockBuf[0x103] = 0x42;
mockBuf[0x420] = 0x15;
mockBuf[0x422] = 0x01;
mockBuf[0x423] = 0x22;
mockBuf[0x220] = 0x24;
mockBuf[0x222] = 0x01;
mockBuf[0x223] = 0x32;
mockBuf[0x320] = 0x10;
mockBuf[0x322] = 0x01;
mockBuf[0x323] = 0x40;
mockBuf[0x400] = 0x18;
mockBuf[0x402] = 0x01;
return pciExtendedConfigSpaceSize;
}
ssize_t preadMockHeaderFailure(int fd, void *buf, size_t count, off_t offset) {
return pciExtendedConfigSpaceSize;
}
ssize_t preadMockInvalidPos(int fd, void *buf, size_t count, off_t offset) {
uint8_t *mockBuf = static_cast<uint8_t *>(buf);
// Sample config values
mockBuf[0x100] = 0x0e;
mockBuf[0x102] = 0x01;
mockBuf[0x420] = 0x15;
mockBuf[0x422] = 0x01;
mockBuf[0x423] = 0x22;
mockBuf[0x220] = 0x24;
mockBuf[0x222] = 0x01;
mockBuf[0x223] = 0x32;
mockBuf[0x320] = 0x10;
mockBuf[0x322] = 0x01;
mockBuf[0x323] = 0x40;
mockBuf[0x400] = 0x18;
mockBuf[0x402] = 0x01;
return pciExtendedConfigSpaceSize;
}
ssize_t preadMockLoop(int fd, void *buf, size_t count, off_t offset) {
uint8_t *mockBuf = static_cast<uint8_t *>(buf);
// Sample config values
mockBuf[0x100] = 0x0e;
mockBuf[0x102] = 0x01;
mockBuf[0x103] = 0x42;
mockBuf[0x420] = 0x16;
mockBuf[0x422] = 0x01;
mockBuf[0x423] = 0x42;
mockBuf[0x220] = 0x24;
mockBuf[0x222] = 0x01;
mockBuf[0x223] = 0x32;
mockBuf[0x320] = 0x10;
mockBuf[0x322] = 0x01;
mockBuf[0x323] = 0x40;
mockBuf[0x400] = 0x18;
mockBuf[0x402] = 0x01;
return pciExtendedConfigSpaceSize;
}
ssize_t preadMockFailure(int fd, void *buf, size_t count, off_t offset) {
return -1;
}
struct MockMemoryManagerPci : public MemoryManagerMock {
MockMemoryManagerPci(NEO::ExecutionEnvironment &executionEnvironment) : MemoryManagerMock(const_cast<NEO::ExecutionEnvironment &>(executionEnvironment)) {}
@@ -90,6 +177,8 @@ class ZesPciFixture : public ::testing::Test {
.WillByDefault(::testing::Invoke(pSysfsAccess.get(), &Mock<PciSysfsAccess>::getValStringRealPath));
ON_CALL(*pSysfsAccess.get(), read(_, Matcher<double &>(_)))
.WillByDefault(::testing::Invoke(pSysfsAccess.get(), &Mock<PciSysfsAccess>::getValDouble));
ON_CALL(*pSysfsAccess.get(), isRootUser())
.WillByDefault(::testing::Invoke(pSysfsAccess.get(), &Mock<PciSysfsAccess>::checkRootUser));
ON_CALL(*pfsAccess.get(), read(_, Matcher<double &>(_)))
.WillByDefault(::testing::Invoke(pfsAccess.get(), &Mock<PcifsAccess>::getValDouble));
ON_CALL(*pfsAccess.get(), read(_, Matcher<int32_t &>(_)))
@@ -98,7 +187,14 @@ class ZesPciFixture : public ::testing::Test {
pOsPciPrev = pPciImp->pOsPci;
pPciImp->pOsPci = nullptr;
memoryManager->localMemorySupported[0] = 0;
pPciImp->init();
PublicLinuxPciImp *pLinuxPciImp = new PublicLinuxPciImp(pOsSysman);
pLinuxPciImp->openFunction = openMock;
pLinuxPciImp->closeFunction = closeMock;
pLinuxPciImp->preadFunction = preadMock;
pLinuxPciImp->pciExtendedConfigRead();
pPciImp->pOsPci = static_cast<OsPci *>(pLinuxPciImp);
pPciImp->pciGetStaticFields();
}
void TearDown() override {
@@ -146,7 +242,7 @@ TEST_F(ZesPciFixture, GivenValidSysmanHandleWhenCallingzetSysmanPciGetProperties
EXPECT_NE(properties.maxSpeed.maxBandwidth, propertiesBefore.maxSpeed.maxBandwidth);
}
TEST_F(ZesPciFixture, GivenValidSysmanHandleWhenSettingLmemSupportAndCallingzetSysmanPciGetPropertiesThenVerifyzetSysmanPciGetPropertiesCallSucceeds) {
TEST_F(ZesPciFixture, GivenValidSysmanHandleWhenSettingLmemSupportAndCallingzetSysmanPciGetPropertiesThenVerifyApiCallSucceeds) {
zes_pci_properties_t properties, propertiesBefore;
memoryManager->localMemorySupported[0] = 1;
pPciImp->init();
@@ -211,6 +307,81 @@ TEST_F(ZesPciFixture, GivenValidSysmanHandleWhenCallingzetSysmanPciGetBarsThenVe
}
}
TEST_F(ZesPciFixture, GivenValidSysmanHandleWhenInitializingPciAndPciConfigOpenFailsThenResizableBarSupportWillBeFalse) {
OsPci *pOsPciOriginal = pPciImp->pOsPci;
PublicLinuxPciImp *pLinuxPciImpTemp = new PublicLinuxPciImp(pOsSysman);
pLinuxPciImpTemp->openFunction = openMockReturnFailure;
pLinuxPciImpTemp->closeFunction = closeMock;
pLinuxPciImpTemp->preadFunction = preadMock;
pLinuxPciImpTemp->pciExtendedConfigRead();
pPciImp->pOsPci = static_cast<OsPci *>(pLinuxPciImpTemp);
pPciImp->pciGetStaticFields();
EXPECT_FALSE(pPciImp->pOsPci->resizableBarSupported());
delete pLinuxPciImpTemp;
pPciImp->pOsPci = pOsPciOriginal;
}
TEST_F(ZesPciFixture, GivenValidSysmanHandleWhenInitializingPciAndPciConfigReadFailsThenResizableBarSupportWillBeFalse) {
OsPci *pOsPciOriginal = pPciImp->pOsPci;
PublicLinuxPciImp *pLinuxPciImpTemp = new PublicLinuxPciImp(pOsSysman);
pLinuxPciImpTemp->openFunction = openMock;
pLinuxPciImpTemp->closeFunction = closeMock;
pLinuxPciImpTemp->preadFunction = preadMockFailure;
pLinuxPciImpTemp->pciExtendedConfigRead();
pPciImp->pOsPci = static_cast<OsPci *>(pLinuxPciImpTemp);
pPciImp->pciGetStaticFields();
EXPECT_FALSE(pPciImp->pOsPci->resizableBarSupported());
delete pLinuxPciImpTemp;
pPciImp->pOsPci = pOsPciOriginal;
}
TEST_F(ZesPciFixture, GivenSysmanHandleWhenCheckForResizableBarSupportAndHeaderFieldNotPresentThenResizableBarSupportFalseReturned) {
OsPci *pOsPciOriginal = pPciImp->pOsPci;
PublicLinuxPciImp *pLinuxPciImpTemp = new PublicLinuxPciImp(pOsSysman);
pLinuxPciImpTemp->openFunction = openMock;
pLinuxPciImpTemp->closeFunction = closeMock;
pLinuxPciImpTemp->preadFunction = preadMockHeaderFailure;
pLinuxPciImpTemp->pciExtendedConfigRead();
pPciImp->pOsPci = static_cast<OsPci *>(pLinuxPciImpTemp);
pPciImp->pciGetStaticFields();
EXPECT_FALSE(pPciImp->pOsPci->resizableBarSupported());
delete pLinuxPciImpTemp;
pPciImp->pOsPci = pOsPciOriginal;
}
TEST_F(ZesPciFixture, GivenSysmanHandleWhenCheckForResizableBarSupportAndCapabilityLinkListIsBrokenThenResizableBarSupportFalseReturned) {
OsPci *pOsPciOriginal = pPciImp->pOsPci;
PublicLinuxPciImp *pLinuxPciImpTemp = new PublicLinuxPciImp(pOsSysman);
pLinuxPciImpTemp->openFunction = openMock;
pLinuxPciImpTemp->closeFunction = closeMock;
pLinuxPciImpTemp->preadFunction = preadMockInvalidPos;
pLinuxPciImpTemp->pciExtendedConfigRead();
pPciImp->pOsPci = static_cast<OsPci *>(pLinuxPciImpTemp);
pPciImp->pciGetStaticFields();
EXPECT_FALSE(pPciImp->pOsPci->resizableBarSupported());
delete pLinuxPciImpTemp;
pPciImp->pOsPci = pOsPciOriginal;
}
TEST_F(ZesPciFixture, GivenSysmanHandleWhenCheckForResizableBarSupportAndIfRebarCapabilityNotPresentThenResizableBarSupportFalseReturned) {
OsPci *pOsPciOriginal = pPciImp->pOsPci;
PublicLinuxPciImp *pLinuxPciImpTemp = new PublicLinuxPciImp(pOsSysman);
pLinuxPciImpTemp->openFunction = openMock;
pLinuxPciImpTemp->closeFunction = closeMock;
pLinuxPciImpTemp->preadFunction = preadMockLoop;
pLinuxPciImpTemp->pciExtendedConfigRead();
pPciImp->pOsPci = static_cast<OsPci *>(pLinuxPciImpTemp);
pPciImp->pciGetStaticFields();
EXPECT_FALSE(pPciImp->pOsPci->resizableBarSupported());
delete pLinuxPciImpTemp;
pPciImp->pOsPci = pOsPciOriginal;
}
TEST_F(ZesPciFixture, GivenValidSysmanHandleWhenCallingzetSysmanPciGetBarsThenVerifyzetSysmanPciGetBarsCallSucceedsWith1_2Extension) {
uint32_t count = 0;
EXPECT_EQ(ZE_RESULT_SUCCESS, zesDevicePciGetBars(device, &count, nullptr));
@@ -234,7 +405,7 @@ TEST_F(ZesPciFixture, GivenValidSysmanHandleWhenCallingzetSysmanPciGetBarsThenVe
EXPECT_NE(pBarProps[i].size, 0u);
}
EXPECT_EQ(props1_2->resizableBarSupported, false);
EXPECT_EQ(props1_2->resizableBarSupported, true);
EXPECT_EQ(props1_2->resizableBarEnabled, false);
EXPECT_LE(props1_2->type, ZES_PCI_BAR_TYPE_MEM);
EXPECT_NE(props1_2->base, 0u);
@@ -299,7 +470,7 @@ TEST_F(ZesPciFixture, GivenValidSysmanHandleWhenCallingzetSysmanPciGetBarsThenVe
delete[] pBarProps;
}
TEST_F(ZesPciFixture, GivenValidSysmanHandleWhenCallingzetSysmanPciGetBarsThenVerifyzetSysmanPciGetBarsCallSucceedsWith1_2ExtensionWithWrongtypeNullPtr) {
TEST_F(ZesPciFixture, GivenValidSysmanHandleWhenCallingzetSysmanPciGetBarsThenVerifyApiCallSucceedsWith1_2ExtensionWithWrongtypeNullPtr) {
uint32_t count = 0;
EXPECT_EQ(ZE_RESULT_SUCCESS, zesDevicePciGetBars(device, &count, nullptr));
EXPECT_NE(count, 0u);