intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2026-01-05 17:20:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

SysMan:fix device reset

Browse Source 
The distance from the SGUnit to the  Root port is constant.
calculating the Rootport and Cardbus based on this observation.
the root port and card bus are used by Warmreset function to
preserve the PCI config space.

Resolves: LOCI-2899

Signed-off-by: Vilvaraj, T J Vivek <t.j.vivek.vilvaraj@intel.com>
This commit is contained in:
Vilvaraj, T J Vivek
2022-06-16 13:02:42 +00:00
committed by Compute-Runtime-Automation
parent 82d3901978
commit 894f90f89e
13 changed files with 95 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
level_zero/tools/source/sysman/global_operations/linux/os_global_operations_imp.cpp
View File

@@ -121,7 +121,6 @@ ze_result_t LinuxGlobalOperationsImp::reset(ze_bool_t force) {
result = pProcfsAccess->listProcesses(processes);
if (ZE_RESULT_SUCCESS != result) {
return result;
}
for (auto &&pid : processes) {
@@ -137,7 +136,6 @@ ze_result_t LinuxGlobalOperationsImp::reset(ze_bool_t force) {
} else {
// Device is in use by another process.
// Don't reset while in use.
return ZE_RESULT_ERROR_HANDLE_OBJECT_IN_USE;
}
}
@@ -149,14 +147,12 @@ ze_result_t LinuxGlobalOperationsImp::reset(ze_bool_t force) {
if (!(deviceProperties.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED)) {
result = pSysfsAccess->unbindDevice(resetName);
if (ZE_RESULT_SUCCESS != result) {
return result;
}
result = pLinuxSysmanImp->osWarmReset();
if (ZE_RESULT_SUCCESS == result) {
return pLinuxSysmanImp->initDevice();
}
return result;
}
@@ -175,7 +171,6 @@ ze_result_t LinuxGlobalOperationsImp::reset(ze_bool_t force) {
// Unbind the device from the kernel driver.
result = pSysfsAccess->unbindDevice(resetName);
if (ZE_RESULT_SUCCESS != result) {
return result;
}
@@ -183,7 +178,6 @@ ze_result_t LinuxGlobalOperationsImp::reset(ze_bool_t force) {
// after we check, kill them here.
result = pProcfsAccess->listProcesses(processes);
if (ZE_RESULT_SUCCESS != result) {
return result;
}
std::vector<::pid_t> deviceUsingPids;
@@ -192,7 +186,6 @@ ze_result_t LinuxGlobalOperationsImp::reset(ze_bool_t force) {
std::vector<int> fds;
pLinuxSysmanImp->getPidFdsForOpenDevice(pProcfsAccess, pSysfsAccess, pid, fds);
if (!fds.empty()) {
// Kill all processes that have the device open.
pProcfsAccess->kill(pid);
deviceUsingPids.push_back(pid);
@@ -210,7 +203,6 @@ ze_result_t LinuxGlobalOperationsImp::reset(ze_bool_t force) {
return ZE_RESULT_ERROR_HANDLE_OBJECT_IN_USE;
}
struct ::timespec timeout = {.tv_sec = 0, .tv_nsec = 1000};
::nanosleep(&timeout, NULL);
end = std::chrono::steady_clock::now();
@@ -220,14 +212,12 @@ ze_result_t LinuxGlobalOperationsImp::reset(ze_bool_t force) {
// Reset the device.
result = pFsAccess->write(resetPath, "1");
if (ZE_RESULT_SUCCESS != result) {
return result;
}
// Rebind the device to the kernel driver.
result = pSysfsAccess->bindDevice(resetName);
if (ZE_RESULT_SUCCESS != result) {
return result;
}

86
level_zero/tools/source/sysman/linux/os_sysman_imp.cpp
View File

@@ -65,13 +65,13 @@ void LinuxSysmanImp::createFwUtilInterface() {
}
ze_result_t LinuxSysmanImp::createPmtHandles() {
std::string realRootPath;
auto result = pSysfsAccess->getRealPath("device", realRootPath);
std::string gtDevicePCIPath;
auto result = pSysfsAccess->getRealPath("device", gtDevicePCIPath);
if (ZE_RESULT_SUCCESS != result) {
return result;
}
auto rootPciPathOfGpuDevice = getPciRootPortDirectoryPath(realRootPath);
PlatformMonitoringTech::create(pParentSysmanDeviceImp->deviceHandles, pFsAccess, rootPciPathOfGpuDevice, mapOfSubDeviceIdToPmtObject);
auto gpuUpstreamPortPath = getPciCardBusDirectoryPath(gtDevicePCIPath);
PlatformMonitoringTech::create(pParentSysmanDeviceImp->deviceHandles, pFsAccess, gpuUpstreamPortPath, mapOfSubDeviceIdToPmtObject);
return result;
}
@@ -136,12 +136,9 @@ SysmanDeviceImp *LinuxSysmanImp::getSysmanDeviceImp() {
return pParentSysmanDeviceImp;
}
std::string LinuxSysmanImp::getPciRootPortDirectoryPath(std::string realPciPath) {
static std::string modifyPathOnLevel(std::string realPciPath, uint8_t nLevel) {
size_t loc;
// we need to change the absolute path to two levels up to get
// the Discrete card's root port.
// the root port is always at a fixed distance as defined in HW
uint8_t nLevel = 2;
// we need to change the absolute path to 'nLevel' levels up
while (nLevel > 0) {
loc = realPciPath.find_last_of('/');
if (loc == std::string::npos) {
@@ -152,38 +149,36 @@ std::string LinuxSysmanImp::getPciRootPortDirectoryPath(std::string realPciPath)
}
return realPciPath;
}
static std::string modifyPathOnLevel(std::string path, uint8_t level) {
size_t loc = 0;
size_t count = 0;
std::string modifiedPath(path);
uint8_t nLevel = level;
do {
loc = path.find_first_of('/');
count = count + loc;
if (loc == std::string::npos) {
break;
}
path = path.substr(loc + 1, path.size());
nLevel--;
} while (nLevel > 0);
if (nLevel == 0) {
modifiedPath = modifiedPath.substr(0, (count + level - 1)); // need to adjust for last '/' that the code encounters
}
return modifiedPath;
}
std::string LinuxSysmanImp::getPciRootPortDirectoryPathForReset(std::string realPciPath) {
std::string getPciRootPortDirectoryPath(std::string realPciPath) {
// the rootport is always the first pci folder after the pcie slot.
// +-[0000:89]-+-00.0
// | +-00.1
// | +-00.2
// | +-00.4
// | \-02.0-[8a-8e]----00.0-[8b-8e]--+-01.0-[8c-8d]----00.0
// | \-02.0-[8e]--+-00.0
// | +-00.1
// | \-00.2
// /sys/devices/pci0000:89/0000:89:02.0/0000:8a:00.0/0000:8b:01.0/0000:8c:00.0
// '/sys/devices/pci0000:89/0000:89:02.0/' will always be the same distance.
return modifyPathOnLevel(realPciPath, 5);
// from 0000:8c:00.0 i.e the 3rd PCI address from the gt tile
return modifyPathOnLevel(realPciPath, 3);
}
std::string LinuxSysmanImp::getPciCardBusDirectoryPath(std::string realPciPath) {
// the cardbus is always the second pci folder after the pcie slot.
// +-[0000:89]-+-00.0
// | +-00.1
// | +-00.2
// | +-00.4
// | \-02.0-[8a-8e]----00.0-[8b-8e]--+-01.0-[8c-8d]----00.0
// | \-02.0-[8e]--+-00.0
// | +-00.1
// | \-00.2
// /sys/devices/pci0000:89/0000:89:02.0/0000:8a:00.0/0000:8b:01.0/0000:8c:00.0
// '/sys/devices/pci0000:89/0000:89:02.0/0000:8a:00.0/' will always be the same distance.
return modifyPathOnLevel(realPciPath, 6);
// from 0000:8c:00.0 i.e the 2nd PCI address from the gt tile.
return modifyPathOnLevel(realPciPath, 2);
}
PlatformMonitoringTech *LinuxSysmanImp::getPlatformMonitoringTechAccess(uint32_t subDeviceId) {
@@ -274,7 +269,7 @@ void LinuxSysmanImp::releaseDeviceResources() {
executionEnvironment = devicePtr->getNEODevice()->getExecutionEnvironment();
devicePciBdf = devicePtr->getNEODevice()->getRootDeviceEnvironment().osInterface->getDriverModel()->as<NEO::Drm>()->getPciPath();
rootDeviceIndex = devicePtr->getNEODevice()->getRootDeviceIndex();
pSysfsAccess->getRealPath(deviceDir, gtDevicePath);
releaseSysmanDeviceResources();
auto device = static_cast<DeviceImp *>(getDeviceHandle());
executionEnvironment = device->getNEODevice()->getExecutionEnvironment();
@@ -320,20 +315,14 @@ ze_result_t LinuxSysmanImp::initDevice() {
// in the bridge control register in the PCI configuration space of the bridge port upstream of the device.
ze_result_t LinuxSysmanImp::osWarmReset() {
std::string rootPortPath;
std::string realRootPath;
ze_result_t result = pSysfsAccess->getRealPath(deviceDir, realRootPath);
if (ZE_RESULT_SUCCESS != result) {
return result;
}
std::string cardBusPath = getPciCardBusDirectoryPath(realRootPath);
result = pFsAccess->write(cardBusPath + '/' + "remove", "1");
std::string cardBusPath = getPciCardBusDirectoryPath(gtDevicePath);
ze_result_t result = pFsAccess->write(cardBusPath + '/' + "remove", "1");
if (ZE_RESULT_SUCCESS != result) {
return result;
}
this->pSleepFunctionSecs(10); // Sleep for 10seconds to make sure that the config spaces of all devices are saved correctly
rootPortPath = getPciRootPortDirectoryPathForReset(realRootPath);
rootPortPath = getPciRootPortDirectoryPath(gtDevicePath);
int fd, ret = 0;
unsigned int offset = PCI_BRIDGE_CONTROL; // Bridge control offset in Header of PCI config space
@@ -370,19 +359,14 @@ std::string LinuxSysmanImp::getAddressFromPath(std::string &rootPortPath) {
}
ze_result_t LinuxSysmanImp::osColdReset() {
const std::string slotPath("/sys/bus/pci/slots/"); // holds the directories matching to the number of slots in the PC
std::string cardBusPath; // will hold the PCIe Root port directory path (the address of the PCIe slot).
std::string realRootPath; // will hold the absolute real path (not symlink) to the selected Device
ze_result_t result = pSysfsAccess->getRealPath(deviceDir, realRootPath); // e.g realRootPath=/sys/devices/pci0000:89/0000:89:02.0/0000:8a:00.0/0000:8b:01.0/0000:8c:00.0
if (ZE_RESULT_SUCCESS != result) {
return result;
}
const std::string slotPath("/sys/bus/pci/slots/"); // holds the directories matching to the number of slots in the PC
std::string cardBusPath; // will hold the PCIe Root port directory path (the address of the PCIe slot). // will hold the absolute real path (not symlink) to the selected Device
cardBusPath = getPciCardBusDirectoryPath(realRootPath); // e.g cardBusPath=/sys/devices/pci0000:89/0000:89:02.0/
cardBusPath = getPciCardBusDirectoryPath(gtDevicePath); // e.g cardBusPath=/sys/devices/pci0000:89/0000:89:02.0/
std::string rootAddress = getAddressFromPath(cardBusPath); // e.g rootAddress = 0000:8a:00.0
std::vector<std::string> dir;
result = pFsAccess->listDirectory(slotPath, dir); // get list of slot directories from /sys/bus/pci/slots/
ze_result_t result = pFsAccess->listDirectory(slotPath, dir); // get list of slot directories from /sys/bus/pci/slots/
if (ZE_RESULT_SUCCESS != result) {
return result;
}

3
level_zero/tools/source/sysman/linux/os_sysman_imp.h
View File

@@ -53,8 +53,6 @@ class LinuxSysmanImp : public OsSysman, NEO::NonCopyableOrMovableClass {
PlatformMonitoringTech *getPlatformMonitoringTechAccess(uint32_t subDeviceId);
Device *getDeviceHandle();
SysmanDeviceImp *getSysmanDeviceImp();
std::string getPciRootPortDirectoryPath(std::string realPciPath);
std::string getPciRootPortDirectoryPathForReset(std::string realPciPath);
std::string getPciCardBusDirectoryPath(std::string realPciPath);
void releasePmtObject();
ze_result_t createPmtHandles();
@@ -80,6 +78,7 @@ class LinuxSysmanImp : public OsSysman, NEO::NonCopyableOrMovableClass {
NEO::ExecutionEnvironment *executionEnvironment = nullptr;
bool diagnosticsReset = false;
Device *pDevice = nullptr;
std::string gtDevicePath;
protected:
FsAccess *pFsAccess = nullptr;

29
level_zero/tools/source/sysman/linux/pmt/pmt.cpp
View File

@@ -74,8 +74,8 @@ bool compareTelemNodes(std::string &telemNode1, std::string &telemNode2) {
return indexForTelemNode1 < indexForTelemNode2;
}
// Check if Telemetry node(say /sys/class/intel_pmt/telem1) and rootPciPathOfGpuDevice share same PCI Root port
static bool isValidTelemNode(FsAccess *pFsAccess, const std::string &rootPciPathOfGpuDevice, const std::string sysfsTelemNode) {
// Check if Telemetry node(say /sys/class/intel_pmt/telem1) and gpuUpstreamPortPath share same PCI Root port
static bool isValidTelemNode(FsAccess *pFsAccess, const std::string &gpuUpstreamPortPath, const std::string sysfsTelemNode) {
std::string realPathOfTelemNode;
auto result = pFsAccess->getRealPath(sysfsTelemNode, realPathOfTelemNode);
if (result != ZE_RESULT_SUCCESS) {
@@ -83,14 +83,15 @@ static bool isValidTelemNode(FsAccess *pFsAccess, const std::string &rootPciPath
}
// Example: If
// rootPciPathOfGpuDevice = "/sys/devices/pci0000:89/0000:89:02.0/0000:8a:00.0";
// gpuUpstreamPortPath = "/sys/devices/pci0000:89/0000:89:02.0/0000:8a:00.0";
// realPathOfTelemNode = "/sys/devices/pci0000:89/0000:89:02.0/0000:8a:00.0/0000:8b:02.0/0000:8e:00.1/pmt_telemetry.1.auto/intel_pmt/telem1";
// As rootPciPathOfGpuDevice is a substring og realPathOfTelemNode , hence both sysfs telemNode and GPU device share same PCI Root.
// As gpuUpstreamPortPath is a substring of realPathOfTelemNode , hence both sysfs telemNode and GPU device share same PCI Root.
// the PMT is part of the OOBMSM sitting on a switch port 0000:8b:02.0 attached to the upstream port/ Also known as CardBus
// Hence this telem node entry is valid for GPU device.
return (realPathOfTelemNode.compare(0, rootPciPathOfGpuDevice.size(), rootPciPathOfGpuDevice) == 0);
return (realPathOfTelemNode.compare(0, gpuUpstreamPortPath.size(), gpuUpstreamPortPath) == 0);
}
ze_result_t PlatformMonitoringTech::enumerateRootTelemIndex(FsAccess *pFsAccess, std::string &rootPciPathOfGpuDevice) {
ze_result_t PlatformMonitoringTech::enumerateRootTelemIndex(FsAccess *pFsAccess, std::string &gpuUpstreamPortPath) {
std::vector<std::string> listOfTelemNodes;
auto result = pFsAccess->listDirectory(baseTelemSysFS, listOfTelemNodes);
if (ZE_RESULT_SUCCESS != result) {
@@ -111,7 +112,7 @@ ze_result_t PlatformMonitoringTech::enumerateRootTelemIndex(FsAccess *pFsAccess,
// Then listOfTelemNodes would contain telem1, telem2, telem3
std::sort(listOfTelemNodes.begin(), listOfTelemNodes.end(), compareTelemNodes); // sort listOfTelemNodes, to arange telem nodes in ascending order
for (const auto &telemNode : listOfTelemNodes) {
if (isValidTelemNode(pFsAccess, rootPciPathOfGpuDevice, baseTelemSysFS + "/" + telemNode)) {
if (isValidTelemNode(pFsAccess, gpuUpstreamPortPath, baseTelemSysFS + "/" + telemNode)) {
auto indexString = telemNode.substr(telem.size(), telemNode.size());
rootDeviceTelemNodeIndex = stoi(indexString); // if telemNode is telemN, then rootDeviceTelemNodeIndex = N
return ZE_RESULT_SUCCESS;
@@ -120,7 +121,7 @@ ze_result_t PlatformMonitoringTech::enumerateRootTelemIndex(FsAccess *pFsAccess,
return ZE_RESULT_ERROR_DEPENDENCY_UNAVAILABLE;
}
ze_result_t PlatformMonitoringTech::init(FsAccess *pFsAccess, const std::string &rootPciPathOfGpuDevice) {
ze_result_t PlatformMonitoringTech::init(FsAccess *pFsAccess, const std::string &gpuUpstreamPortPath) {
std::string telemNode = telem + std::to_string(rootDeviceTelemNodeIndex);
if (isSubdevice) {
uint32_t telemNodeIndex = 0;
@@ -131,7 +132,7 @@ ze_result_t PlatformMonitoringTech::init(FsAccess *pFsAccess, const std::string
telemNode = telem + std::to_string(telemNodeIndex);
}
std::string baseTelemSysFSNode = baseTelemSysFS + "/" + telemNode;
if (!isValidTelemNode(pFsAccess, rootPciPathOfGpuDevice, baseTelemSysFSNode)) {
if (!isValidTelemNode(pFsAccess, gpuUpstreamPortPath, baseTelemSysFSNode)) {
return ZE_RESULT_ERROR_DEPENDENCY_UNAVAILABLE;
}
@@ -172,9 +173,9 @@ PlatformMonitoringTech::PlatformMonitoringTech(FsAccess *pFsAccess, ze_bool_t on
}
void PlatformMonitoringTech::doInitPmtObject(FsAccess *pFsAccess, uint32_t subdeviceId, PlatformMonitoringTech *pPmt,
const std::string &rootPciPathOfGpuDevice,
const std::string &gpuUpstreamPortPath,
std::map<uint32_t, L0::PlatformMonitoringTech *> &mapOfSubDeviceIdToPmtObject) {
if (pPmt->init(pFsAccess, rootPciPathOfGpuDevice) == ZE_RESULT_SUCCESS) {
if (pPmt->init(pFsAccess, gpuUpstreamPortPath) == ZE_RESULT_SUCCESS) {
mapOfSubDeviceIdToPmtObject.emplace(subdeviceId, pPmt);
return;
}
@@ -182,9 +183,9 @@ void PlatformMonitoringTech::doInitPmtObject(FsAccess *pFsAccess, uint32_t subde
}
void PlatformMonitoringTech::create(const std::vector<ze_device_handle_t> &deviceHandles,
FsAccess *pFsAccess, std::string &rootPciPathOfGpuDevice,
FsAccess *pFsAccess, std::string &gpuUpstreamPortPath,
std::map<uint32_t, L0::PlatformMonitoringTech *> &mapOfSubDeviceIdToPmtObject) {
if (ZE_RESULT_SUCCESS == PlatformMonitoringTech::enumerateRootTelemIndex(pFsAccess, rootPciPathOfGpuDevice)) {
if (ZE_RESULT_SUCCESS == PlatformMonitoringTech::enumerateRootTelemIndex(pFsAccess, gpuUpstreamPortPath)) {
for (const auto &deviceHandle : deviceHandles) {
uint32_t subdeviceId = 0;
ze_bool_t onSubdevice = false;
@@ -192,7 +193,7 @@ void PlatformMonitoringTech::create(const std::vector<ze_device_handle_t> &devic
auto pPmt = new PlatformMonitoringTech(pFsAccess, onSubdevice, subdeviceId);
UNRECOVERABLE_IF(nullptr == pPmt);
PlatformMonitoringTech::doInitPmtObject(pFsAccess, subdeviceId, pPmt,
rootPciPathOfGpuDevice, mapOfSubDeviceIdToPmtObject);
gpuUpstreamPortPath, mapOfSubDeviceIdToPmtObject);
}
}
}

8
level_zero/tools/source/sysman/linux/pmt/pmt.h
View File

@@ -27,9 +27,9 @@ class PlatformMonitoringTech : NEO::NonCopyableOrMovableClass {
virtual ze_result_t readValue(const std::string key, uint32_t &value);
virtual ze_result_t readValue(const std::string key, uint64_t &value);
static ze_result_t enumerateRootTelemIndex(FsAccess *pFsAccess, std::string &rootPciPathOfGpuDevice);
static ze_result_t enumerateRootTelemIndex(FsAccess *pFsAccess, std::string &gpuUpstreamPortPath);
static void create(const std::vector<ze_device_handle_t> &deviceHandles,
FsAccess *pFsAccess, std::string &rootPciPathOfGpuDevice,
FsAccess *pFsAccess, std::string &gpuUpstreamPortPath,
std::map<uint32_t, L0::PlatformMonitoringTech *> &mapOfSubDeviceIdToPmtObject);
protected:
@@ -37,8 +37,8 @@ class PlatformMonitoringTech : NEO::NonCopyableOrMovableClass {
std::string telemetryDeviceEntry{};
std::map<std::string, uint64_t> keyOffsetMap;
ze_result_t getKeyOffsetMap(std::string guid, std::map<std::string, uint64_t> &keyOffsetMap);
ze_result_t init(FsAccess *pFsAccess, const std::string &rootPciPathOfGpuDevice);
static void doInitPmtObject(FsAccess *pFsAccess, uint32_t subdeviceId, PlatformMonitoringTech *pPmt, const std::string &rootPciPathOfGpuDevice,
ze_result_t init(FsAccess *pFsAccess, const std::string &gpuUpstreamPortPath);
static void doInitPmtObject(FsAccess *pFsAccess, uint32_t subdeviceId, PlatformMonitoringTech *pPmt, const std::string &gpuUpstreamPortPath,
std::map<uint32_t, L0::PlatformMonitoringTech *> &mapOfSubDeviceIdToPmtObject);
decltype(&NEO::SysCalls::open) openFunction = NEO::SysCalls::open;
decltype(&NEO::SysCalls::close) closeFunction = NEO::SysCalls::close;

2
level_zero/tools/source/sysman/pci/linux/os_pci_imp.cpp
View File

@@ -274,7 +274,7 @@ void LinuxPciImp::pciCardBusConfigRead() {
std::string pciConfigNode;
std::string rootPortPath;
pSysfsAccess->getRealPath(deviceDir, pciConfigNode);
rootPortPath = pLinuxSysmanImp->getPciRootPortDirectoryPath(pciConfigNode);
rootPortPath = pLinuxSysmanImp->getPciCardBusDirectoryPath(pciConfigNode);
pciConfigNode = rootPortPath + "/config";
int fdConfig = -1;
fdConfig = this->openFunction(pciConfigNode.c_str(), O_RDONLY);