mirror of
https://github.com/intel/compute-runtime.git
synced 2026-01-03 23:03:02 +08:00
fix(sysman): Changes in Power Module w.r.t Xe
- Added support for min, max and default limits w.r.t Xe driver. - Added support for getPropertiesExt, getLimitsExt, setLimitsExt. - The Unit conversion logic defined in the SysmanKmdInterface class has been used. Related-To: NEO-9979 Signed-off-by: Bari, Pratik <pratik.bari@intel.com>
This commit is contained in:
committed by
Compute-Runtime-Automation
parent
5d09350ff9
commit
107156033b
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (C) 2023 Intel Corporation
|
||||
* Copyright (C) 2023-2024 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
@@ -10,6 +10,7 @@
|
||||
#include "shared/source/debug_settings/debug_settings_manager.h"
|
||||
|
||||
#include "level_zero/sysman/source/shared/linux/pmt/sysman_pmt.h"
|
||||
#include "level_zero/sysman/source/shared/linux/product_helper/sysman_product_helper.h"
|
||||
#include "level_zero/sysman/source/shared/linux/sysman_fs_access_interface.h"
|
||||
#include "level_zero/sysman/source/shared/linux/sysman_kmd_interface.h"
|
||||
#include "level_zero/sysman/source/shared/linux/zes_os_sysman_imp.h"
|
||||
@@ -18,12 +19,28 @@
|
||||
namespace L0 {
|
||||
namespace Sysman {
|
||||
|
||||
const std::string LinuxPowerImp::sustainedPowerLimitEnabled("power1_max_enable");
|
||||
const std::string LinuxPowerImp::burstPowerLimitEnabled("power1_cap_enable");
|
||||
const std::string LinuxPowerImp::burstPowerLimit("power1_cap");
|
||||
const std::string LinuxPowerImp::defaultPowerLimit("power_default_limit");
|
||||
const std::string LinuxPowerImp::minPowerLimit("power_min_limit");
|
||||
const std::string LinuxPowerImp::maxPowerLimit("power_max_limit");
|
||||
class LinuxPowerImp::PowerLimitRestorer : NEO::NonCopyableOrMovableClass {
|
||||
public:
|
||||
PowerLimitRestorer(L0::Sysman::SysFsAccessInterface *pSysfsAccess, std::string powerLimit) : pSysfsAccess(pSysfsAccess), powerLimit(powerLimit) {
|
||||
result = pSysfsAccess->read(powerLimit, powerLimitValue);
|
||||
}
|
||||
|
||||
~PowerLimitRestorer() {
|
||||
if (result == ZE_RESULT_SUCCESS) {
|
||||
result = pSysfsAccess->write(powerLimit, powerLimitValue);
|
||||
DEBUG_BREAK_IF(result != ZE_RESULT_SUCCESS);
|
||||
}
|
||||
}
|
||||
operator ze_result_t() const {
|
||||
return result;
|
||||
}
|
||||
|
||||
protected:
|
||||
ze_result_t result = ZE_RESULT_ERROR_UNINITIALIZED;
|
||||
SysFsAccessInterface *pSysfsAccess = nullptr;
|
||||
std::string powerLimit = {};
|
||||
uint64_t powerLimitValue = 0;
|
||||
};
|
||||
|
||||
ze_result_t LinuxPowerImp::getProperties(zes_power_properties_t *pProperties) {
|
||||
pProperties->onSubdevice = isSubdevice;
|
||||
@@ -34,27 +51,111 @@ ze_result_t LinuxPowerImp::getProperties(zes_power_properties_t *pProperties) {
|
||||
pProperties->minLimit = -1;
|
||||
pProperties->maxLimit = -1;
|
||||
|
||||
uint32_t val = 0;
|
||||
auto result = pSysfsAccess->read(intelGraphicsHwmonDir + "/" + defaultPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS == result) {
|
||||
pProperties->defaultLimit = static_cast<int32_t>(val / milliFactor); // need to convert from microwatt to milliwatt
|
||||
if (isSubdevice) {
|
||||
return ZE_RESULT_SUCCESS;
|
||||
}
|
||||
|
||||
result = pSysfsAccess->read(intelGraphicsHwmonDir + "/" + minPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS == result && val != 0) {
|
||||
pProperties->minLimit = static_cast<int32_t>(val / milliFactor); // need to convert from microwatt to milliwatt
|
||||
auto result = getDefaultLimit(pProperties->defaultLimit);
|
||||
if (result != ZE_RESULT_SUCCESS) {
|
||||
return result;
|
||||
}
|
||||
|
||||
result = pSysfsAccess->read(intelGraphicsHwmonDir + "/" + maxPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS == result && val != std::numeric_limits<uint32_t>::max()) {
|
||||
pProperties->maxLimit = static_cast<int32_t>(val / milliFactor); // need to convert from microwatt to milliwatt
|
||||
auto powerLimitRestorer = L0::Sysman::LinuxPowerImp::PowerLimitRestorer(pSysfsAccess, sustainedPowerLimit);
|
||||
if (powerLimitRestorer != ZE_RESULT_SUCCESS) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): Failed to read %s and returning error:0x%x \n", __FUNCTION__, sustainedPowerLimit.c_str(), getErrorCode(powerLimitRestorer));
|
||||
return getErrorCode(powerLimitRestorer);
|
||||
}
|
||||
return ZE_RESULT_SUCCESS;
|
||||
|
||||
result = getMinLimit(pProperties->minLimit);
|
||||
if (result != ZE_RESULT_SUCCESS) {
|
||||
return result;
|
||||
}
|
||||
|
||||
return getMaxLimit(pProperties->maxLimit);
|
||||
}
|
||||
|
||||
ze_result_t LinuxPowerImp::getMinLimit(int32_t &minLimit) {
|
||||
// Fw clamps to minimum value if power limit requested to set is less than min limit, Set to 100 micro watt to get min limit
|
||||
uint64_t powerLimit = 100;
|
||||
auto result = pSysfsAccess->write(sustainedPowerLimit, powerLimit);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): Failed to write %s and returning error:0x%x \n", __FUNCTION__, sustainedPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
|
||||
result = pSysfsAccess->read(sustainedPowerLimit, powerLimit);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): Failed to read %s and returning error:0x%x \n", __FUNCTION__, sustainedPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(SysmanKmdInterface::milli, pSysmanKmdInterface->getNativeUnit(SysfsName::sysfsNameSustainedPowerLimit), powerLimit, powerLimit);
|
||||
minLimit = static_cast<int32_t>(powerLimit);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
ze_result_t LinuxPowerImp::getMaxLimit(int32_t &maxLimit) {
|
||||
// Fw clamps to maximum value if power limit requested to set is greater than max limit, Set to max value to get max limit
|
||||
uint64_t powerLimit = std::numeric_limits<int32_t>::max();
|
||||
auto result = pSysfsAccess->write(sustainedPowerLimit, powerLimit);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): Failed to write %s and returning error:0x%x \n", __FUNCTION__, sustainedPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
|
||||
result = pSysfsAccess->read(sustainedPowerLimit, powerLimit);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): Failed to read %s and returning error:0x%x \n", __FUNCTION__, sustainedPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(SysmanKmdInterface::milli, pSysmanKmdInterface->getNativeUnit(SysfsName::sysfsNameSustainedPowerLimit), powerLimit, powerLimit);
|
||||
maxLimit = static_cast<int32_t>(powerLimit);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
ze_result_t LinuxPowerImp::getDefaultLimit(int32_t &defaultLimit) {
|
||||
uint64_t powerLimit = 0;
|
||||
std::string defaultPowerLimit = intelGraphicsHwmonDir + "/" + pSysmanKmdInterface->getSysfsFilePath(SysfsName::sysfsNameDefaultPowerLimit, subdeviceId, false);
|
||||
auto result = pSysfsAccess->read(defaultPowerLimit, powerLimit);
|
||||
if (result != ZE_RESULT_SUCCESS) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), defaultPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(SysmanKmdInterface::milli, pSysmanKmdInterface->getNativeUnit(SysfsName::sysfsNameDefaultPowerLimit), powerLimit, powerLimit);
|
||||
defaultLimit = static_cast<int32_t>(powerLimit);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
ze_result_t LinuxPowerImp::getPropertiesExt(zes_power_ext_properties_t *pExtPoperties) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s() returning UNSUPPORTED_FEATURE \n", __FUNCTION__);
|
||||
return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
|
||||
pExtPoperties->domain = isSubdevice ? ZES_POWER_DOMAIN_PACKAGE : ZES_POWER_DOMAIN_CARD;
|
||||
if (pExtPoperties->defaultLimit) {
|
||||
if (!isSubdevice) {
|
||||
uint64_t val = 0;
|
||||
std::string defaultPowerLimit = intelGraphicsHwmonDir + "/" + pSysmanKmdInterface->getSysfsFilePath(SysfsName::sysfsNameDefaultPowerLimit, subdeviceId, false);
|
||||
ze_result_t result = pSysfsAccess->read(defaultPowerLimit, val);
|
||||
if (result == ZE_RESULT_SUCCESS) {
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(SysmanKmdInterface::milli, pSysmanKmdInterface->getNativeUnit(SysfsName::sysfsNameDefaultPowerLimit), val, val);
|
||||
pExtPoperties->defaultLimit->limit = static_cast<int32_t>(val);
|
||||
} else {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), defaultPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
} else {
|
||||
pExtPoperties->defaultLimit->limit = -1;
|
||||
}
|
||||
pExtPoperties->defaultLimit->limitUnit = ZES_LIMIT_UNIT_POWER;
|
||||
pExtPoperties->defaultLimit->enabledStateLocked = true;
|
||||
pExtPoperties->defaultLimit->intervalValueLocked = true;
|
||||
pExtPoperties->defaultLimit->limitValueLocked = true;
|
||||
pExtPoperties->defaultLimit->source = ZES_POWER_SOURCE_ANY;
|
||||
pExtPoperties->defaultLimit->level = ZES_POWER_LEVEL_UNKNOWN;
|
||||
}
|
||||
return ZE_RESULT_SUCCESS;
|
||||
}
|
||||
|
||||
ze_result_t LinuxPowerImp::getPmtEnergyCounter(zes_power_energy_counter_t *pEnergy) {
|
||||
@@ -66,6 +167,7 @@ ze_result_t LinuxPowerImp::getPmtEnergyCounter(zes_power_energy_counter_t *pEner
|
||||
pEnergy->energy = (energy / fixedPointToJoule) * convertJouleToMicroJoule;
|
||||
return result;
|
||||
}
|
||||
|
||||
ze_result_t LinuxPowerImp::getEnergyCounter(zes_power_energy_counter_t *pEnergy) {
|
||||
pEnergy->timestamp = SysmanDevice::getSysmanTimestamp();
|
||||
std::string energyCounterNode = intelGraphicsHwmonDir + "/" + pSysmanKmdInterface->getSysfsFilePath(SysfsName::sysfsNameEnergyCounterNode, subdeviceId, false);
|
||||
@@ -83,119 +185,67 @@ ze_result_t LinuxPowerImp::getEnergyCounter(zes_power_energy_counter_t *pEnergy)
|
||||
}
|
||||
|
||||
ze_result_t LinuxPowerImp::getLimits(zes_power_sustained_limit_t *pSustained, zes_power_burst_limit_t *pBurst, zes_power_peak_limit_t *pPeak) {
|
||||
ze_result_t result = ZE_RESULT_ERROR_UNKNOWN;
|
||||
uint64_t val = 0;
|
||||
if (pSustained != nullptr) {
|
||||
result = pSysfsAccess->read(intelGraphicsHwmonDir + "/" + sustainedPowerLimitEnabled, val);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), sustainedPowerLimitEnabled.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
pSustained->enabled = static_cast<ze_bool_t>(val);
|
||||
|
||||
if (pSustained->enabled) {
|
||||
ze_result_t result = ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
|
||||
if (!isSubdevice) {
|
||||
uint64_t val = 0;
|
||||
if (pSustained != nullptr) {
|
||||
val = 0;
|
||||
std::string sustainedPowerLimit = intelGraphicsHwmonDir + "/" + pSysmanKmdInterface->getSysfsFilePath(SysfsName::sysfsNameSustainedPowerLimit, subdeviceId, false);
|
||||
result = pSysfsAccess->read(sustainedPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), sustainedPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
val /= milliFactor; // Convert microWatts to milliwatts
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(SysmanKmdInterface::milli, pSysmanKmdInterface->getNativeUnit(SysfsName::sysfsNameSustainedPowerLimit), val, val);
|
||||
pSustained->power = static_cast<int32_t>(val);
|
||||
|
||||
val = 0;
|
||||
std::string sustainedPowerLimitInterval = intelGraphicsHwmonDir + "/" + pSysmanKmdInterface->getSysfsFilePath(SysfsName::sysfsNameSustainedPowerLimitInterval, subdeviceId, false);
|
||||
result = pSysfsAccess->read(sustainedPowerLimitInterval, val);
|
||||
pSustained->enabled = true;
|
||||
pSustained->interval = -1;
|
||||
}
|
||||
if (pBurst != nullptr) {
|
||||
pBurst->power = -1;
|
||||
pBurst->enabled = false;
|
||||
}
|
||||
if (pPeak != nullptr) {
|
||||
result = pSysfsAccess->read(criticalPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), sustainedPowerLimitInterval.c_str(), getErrorCode(result));
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), criticalPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
pSustained->interval = static_cast<int32_t>(val);
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(SysmanKmdInterface::milli, pSysmanKmdInterface->getNativeUnit(SysfsName::sysfsNameCriticalPowerLimit), val, val);
|
||||
pPeak->powerAC = static_cast<int32_t>(val);
|
||||
pPeak->powerDC = -1;
|
||||
}
|
||||
}
|
||||
if (pBurst != nullptr) {
|
||||
result = pSysfsAccess->read(intelGraphicsHwmonDir + "/" + burstPowerLimitEnabled, val);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), burstPowerLimitEnabled.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
pBurst->enabled = static_cast<ze_bool_t>(val);
|
||||
|
||||
if (pBurst->enabled) {
|
||||
result = pSysfsAccess->read(intelGraphicsHwmonDir + "/" + burstPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), burstPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
val /= milliFactor; // Convert microWatts to milliwatts
|
||||
pBurst->power = static_cast<int32_t>(val);
|
||||
}
|
||||
}
|
||||
if (pPeak != nullptr) {
|
||||
pPeak->powerAC = -1;
|
||||
pPeak->powerDC = -1;
|
||||
result = ZE_RESULT_SUCCESS;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
ze_result_t LinuxPowerImp::setLimits(const zes_power_sustained_limit_t *pSustained, const zes_power_burst_limit_t *pBurst, const zes_power_peak_limit_t *pPeak) {
|
||||
ze_result_t result = ZE_RESULT_ERROR_UNKNOWN;
|
||||
int32_t val = 0;
|
||||
if (pSustained != nullptr) {
|
||||
uint64_t isSustainedPowerLimitEnabled = 0;
|
||||
result = pSysfsAccess->read(intelGraphicsHwmonDir + "/" + sustainedPowerLimitEnabled, isSustainedPowerLimitEnabled);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), sustainedPowerLimitEnabled.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
if (isSustainedPowerLimitEnabled != static_cast<uint64_t>(pSustained->enabled)) {
|
||||
result = pSysfsAccess->write(intelGraphicsHwmonDir + "/" + sustainedPowerLimitEnabled, static_cast<int>(pSustained->enabled));
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->write() failed to write into %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), sustainedPowerLimitEnabled.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
isSustainedPowerLimitEnabled = static_cast<uint64_t>(pSustained->enabled);
|
||||
}
|
||||
ze_result_t result = ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
|
||||
if (!isSubdevice) {
|
||||
uint64_t val = 0;
|
||||
|
||||
if (isSustainedPowerLimitEnabled) {
|
||||
val = static_cast<uint32_t>(pSustained->power) * milliFactor; // Convert milliWatts to microwatts
|
||||
std::string sustainedPowerLimit = intelGraphicsHwmonDir + "/" + pSysmanKmdInterface->getSysfsFilePath(SysfsName::sysfsNameSustainedPowerLimit, subdeviceId, false);
|
||||
if (pSustained != nullptr) {
|
||||
val = static_cast<uint64_t>(pSustained->power);
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(pSysmanKmdInterface->getNativeUnit(SysfsName::sysfsNameSustainedPowerLimit), SysmanKmdInterface::milli, val, val);
|
||||
result = pSysfsAccess->write(sustainedPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->write() failed to write into %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), sustainedPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
|
||||
std::string sustainedPowerLimitInterval = intelGraphicsHwmonDir + "/" + pSysmanKmdInterface->getSysfsFilePath(SysfsName::sysfsNameSustainedPowerLimitInterval, subdeviceId, false);
|
||||
result = pSysfsAccess->write(sustainedPowerLimitInterval, pSustained->interval);
|
||||
}
|
||||
if (pPeak != nullptr) {
|
||||
val = static_cast<uint64_t>(pPeak->powerAC);
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(pSysmanKmdInterface->getNativeUnit(SysfsName::sysfsNameCriticalPowerLimit), SysmanKmdInterface::milli, val, val);
|
||||
result = pSysfsAccess->write(criticalPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->write() failed to write into %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), sustainedPowerLimitInterval.c_str(), getErrorCode(result));
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->write() failed to write into %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), criticalPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
}
|
||||
result = ZE_RESULT_SUCCESS;
|
||||
}
|
||||
if (pBurst != nullptr) {
|
||||
result = pSysfsAccess->write(intelGraphicsHwmonDir + "/" + burstPowerLimitEnabled, static_cast<int>(pBurst->enabled));
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->write() failed to write into %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), burstPowerLimitEnabled.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
|
||||
if (pBurst->enabled) {
|
||||
val = static_cast<uint32_t>(pBurst->power) * milliFactor; // Convert milliWatts to microwatts
|
||||
result = pSysfsAccess->write(intelGraphicsHwmonDir + "/" + burstPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->write() failed to write into %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), burstPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
ze_result_t LinuxPowerImp::getEnergyThreshold(zes_energy_threshold_t *pThreshold) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s() returning UNSUPPORTED_FEATURE \n", __FUNCTION__);
|
||||
return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
|
||||
@@ -207,18 +257,98 @@ ze_result_t LinuxPowerImp::setEnergyThreshold(double threshold) {
|
||||
}
|
||||
|
||||
ze_result_t LinuxPowerImp::getLimitsExt(uint32_t *pCount, zes_power_limit_ext_desc_t *pSustained) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s() returning UNSUPPORTED_FEATURE \n", __FUNCTION__);
|
||||
return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
|
||||
ze_result_t result = ZE_RESULT_SUCCESS;
|
||||
if ((*pCount == 0) || (powerLimitCount < *pCount)) {
|
||||
*pCount = powerLimitCount;
|
||||
}
|
||||
|
||||
if (pSustained != nullptr) {
|
||||
uint64_t val = 0;
|
||||
uint8_t count = 0;
|
||||
if (count < *pCount) {
|
||||
result = pSysfsAccess->read(sustainedPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), sustainedPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
|
||||
int32_t interval = 0;
|
||||
result = pSysfsAccess->read(sustainedPowerLimitInterval, interval);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), sustainedPowerLimitInterval.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(SysmanKmdInterface::milli, pSysmanKmdInterface->getNativeUnit(SysfsName::sysfsNameSustainedPowerLimit), val, val);
|
||||
pSustained[count].limit = static_cast<int32_t>(val);
|
||||
pSustained[count].enabledStateLocked = true;
|
||||
pSustained[count].intervalValueLocked = false;
|
||||
pSustained[count].limitValueLocked = false;
|
||||
pSustained[count].source = ZES_POWER_SOURCE_ANY;
|
||||
pSustained[count].level = ZES_POWER_LEVEL_SUSTAINED;
|
||||
pSustained[count].limitUnit = ZES_LIMIT_UNIT_POWER;
|
||||
pSustained[count].interval = interval;
|
||||
count++;
|
||||
}
|
||||
|
||||
if (count < *pCount) {
|
||||
result = pSysfsAccess->read(criticalPowerLimit, val);
|
||||
if (result != ZE_RESULT_SUCCESS) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->read() failed to read %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), criticalPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
pSustained[count].enabledStateLocked = true;
|
||||
pSustained[count].intervalValueLocked = true;
|
||||
pSustained[count].limitValueLocked = false;
|
||||
pSustained[count].source = ZES_POWER_SOURCE_ANY;
|
||||
pSustained[count].level = ZES_POWER_LEVEL_PEAK;
|
||||
pSustained[count].interval = 0;
|
||||
pSustained[count].limit = pSysmanProductHelper->getPowerLimitValue(val);
|
||||
pSustained[count].limitUnit = pSysmanProductHelper->getPowerLimitUnit();
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
ze_result_t LinuxPowerImp::setLimitsExt(uint32_t *pCount, zes_power_limit_ext_desc_t *pSustained) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s() returning UNSUPPORTED_FEATURE \n", __FUNCTION__);
|
||||
return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
|
||||
ze_result_t result = ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
|
||||
if (!isSubdevice) {
|
||||
uint64_t val = 0;
|
||||
for (uint32_t i = 0; i < *pCount; i++) {
|
||||
if (pSustained[i].level == ZES_POWER_LEVEL_SUSTAINED) {
|
||||
val = static_cast<uint64_t>(pSustained[i].limit);
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(pSysmanKmdInterface->getNativeUnit(SysfsName::sysfsNameSustainedPowerLimit), SysmanKmdInterface::milli, val, val);
|
||||
result = pSysfsAccess->write(sustainedPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->write() failed to write into %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), sustainedPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
|
||||
result = pSysfsAccess->write(sustainedPowerLimitInterval, pSustained[i].interval);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->write() failed to write into %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), sustainedPowerLimitInterval.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
} else if (pSustained[i].level == ZES_POWER_LEVEL_PEAK) {
|
||||
val = pSysmanProductHelper->setPowerLimitValue(pSustained[i].limit);
|
||||
result = pSysfsAccess->write(criticalPowerLimit, val);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s(): SysfsAccess->write() failed to write into %s/%s and returning error:0x%x \n", __FUNCTION__, intelGraphicsHwmonDir.c_str(), criticalPowerLimit.c_str(), getErrorCode(result));
|
||||
return getErrorCode(result);
|
||||
}
|
||||
} else {
|
||||
NEO::printDebugString(NEO::debugManager.flags.PrintDebugMessages.get(), stderr, "Error@ %s() returning UNSUPPORTED_FEATURE \n", __FUNCTION__);
|
||||
return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
|
||||
}
|
||||
}
|
||||
result = ZE_RESULT_SUCCESS;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
bool LinuxPowerImp::isHwmonDir(std::string name) {
|
||||
bool LinuxPowerImp::isIntelGraphicsHwmonDir(const std::string &name) {
|
||||
std::string intelGraphicsHwmonName = pSysmanKmdInterface->getHwmonName(subdeviceId, isSubdevice);
|
||||
if (isSubdevice == false && (name == intelGraphicsHwmonName)) {
|
||||
if (name == intelGraphicsHwmonName) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
@@ -226,23 +356,38 @@ bool LinuxPowerImp::isHwmonDir(std::string name) {
|
||||
|
||||
bool LinuxPowerImp::isPowerModuleSupported() {
|
||||
std::vector<std::string> listOfAllHwmonDirs = {};
|
||||
const std::string hwmonDir("device/hwmon");
|
||||
bool hwmonDirExists = false;
|
||||
const std::string hwmonDir("device/hwmon");
|
||||
if (ZE_RESULT_SUCCESS != pSysfsAccess->scanDirEntries(hwmonDir, listOfAllHwmonDirs)) {
|
||||
hwmonDirExists = false;
|
||||
}
|
||||
for (const auto &tempHwmonDirEntry : listOfAllHwmonDirs) {
|
||||
const std::string i915NameFile = hwmonDir + "/" + tempHwmonDirEntry + "/" + "name";
|
||||
const std::string hwmonNameFile = hwmonDir + "/" + tempHwmonDirEntry + "/" + "name";
|
||||
std::string name;
|
||||
if (ZE_RESULT_SUCCESS != pSysfsAccess->read(i915NameFile, name)) {
|
||||
if (ZE_RESULT_SUCCESS != pSysfsAccess->read(hwmonNameFile, name)) {
|
||||
continue;
|
||||
}
|
||||
if (isHwmonDir(name)) {
|
||||
if (isIntelGraphicsHwmonDir(name)) {
|
||||
intelGraphicsHwmonDir = hwmonDir + "/" + tempHwmonDirEntry;
|
||||
hwmonDirExists = true;
|
||||
canControl = true;
|
||||
canControl = isSubdevice ? false : true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!isSubdevice) {
|
||||
uint64_t val = 0;
|
||||
sustainedPowerLimit = intelGraphicsHwmonDir + "/" + pSysmanKmdInterface->getSysfsFilePath(SysfsName::sysfsNameSustainedPowerLimit, subdeviceId, false);
|
||||
criticalPowerLimit = intelGraphicsHwmonDir + "/" + pSysmanKmdInterface->getSysfsFilePath(SysfsName::sysfsNameCriticalPowerLimit, subdeviceId, false);
|
||||
sustainedPowerLimitInterval = intelGraphicsHwmonDir + "/" + pSysmanKmdInterface->getSysfsFilePath(SysfsName::sysfsNameSustainedPowerLimitInterval, subdeviceId, false);
|
||||
if (ZE_RESULT_SUCCESS == pSysfsAccess->read(sustainedPowerLimit, val)) {
|
||||
powerLimitCount++;
|
||||
}
|
||||
|
||||
if (ZE_RESULT_SUCCESS == pSysfsAccess->read(criticalPowerLimit, val)) {
|
||||
powerLimitCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (hwmonDirExists == false) {
|
||||
return (pPmt != nullptr);
|
||||
}
|
||||
@@ -254,6 +399,7 @@ LinuxPowerImp::LinuxPowerImp(OsSysman *pOsSysman, ze_bool_t onSubdevice, uint32_
|
||||
pPmt = pLinuxSysmanImp->getPlatformMonitoringTechAccess(subdeviceId);
|
||||
pSysmanKmdInterface = pLinuxSysmanImp->getSysmanKmdInterface();
|
||||
pSysfsAccess = pSysmanKmdInterface->getSysFsAccess();
|
||||
pSysmanProductHelper = pLinuxSysmanImp->getSysmanProductHelper();
|
||||
}
|
||||
|
||||
OsPower *OsPower::create(OsSysman *pOsSysman, ze_bool_t onSubdevice, uint32_t subdeviceId) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (C) 2023 Intel Corporation
|
||||
* Copyright (C) 2023-2024 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
@@ -19,6 +19,7 @@ namespace Sysman {
|
||||
class PlatformMonitoringTech;
|
||||
class SysFsAccessInterface;
|
||||
class SysmanKmdInterface;
|
||||
class SysmanProductHelper;
|
||||
class LinuxPowerImp : public OsPower, NEO::NonCopyableOrMovableClass {
|
||||
public:
|
||||
ze_result_t getProperties(zes_power_properties_t *pProperties) override;
|
||||
@@ -32,7 +33,7 @@ class LinuxPowerImp : public OsPower, NEO::NonCopyableOrMovableClass {
|
||||
ze_result_t getPropertiesExt(zes_power_ext_properties_t *pExtPoperties) override;
|
||||
|
||||
bool isPowerModuleSupported() override;
|
||||
bool isHwmonDir(std::string name);
|
||||
bool isIntelGraphicsHwmonDir(const std::string &name);
|
||||
ze_result_t getPmtEnergyCounter(zes_power_energy_counter_t *pEnergy);
|
||||
LinuxPowerImp(OsSysman *pOsSysman, ze_bool_t onSubdevice, uint32_t subdeviceId);
|
||||
LinuxPowerImp() = default;
|
||||
@@ -42,22 +43,18 @@ class LinuxPowerImp : public OsPower, NEO::NonCopyableOrMovableClass {
|
||||
PlatformMonitoringTech *pPmt = nullptr;
|
||||
SysFsAccessInterface *pSysfsAccess = nullptr;
|
||||
SysmanKmdInterface *pSysmanKmdInterface = nullptr;
|
||||
SysmanProductHelper *pSysmanProductHelper = nullptr;
|
||||
|
||||
private:
|
||||
std::string intelGraphicsHwmonDir;
|
||||
std::string energyHwmonDir;
|
||||
static const std::string sustainedPowerLimitEnabled;
|
||||
static const std::string sustainedPowerLimit;
|
||||
static const std::string sustainedPowerLimitInterval;
|
||||
static const std::string burstPowerLimitEnabled;
|
||||
static const std::string burstPowerLimit;
|
||||
static const std::string energyCounterNode;
|
||||
static const std::string defaultPowerLimit;
|
||||
static const std::string minPowerLimit;
|
||||
static const std::string maxPowerLimit;
|
||||
std::string intelGraphicsHwmonDir = {};
|
||||
std::string criticalPowerLimit = {};
|
||||
std::string sustainedPowerLimit = {};
|
||||
std::string sustainedPowerLimitInterval = {};
|
||||
bool canControl = false;
|
||||
bool isSubdevice = false;
|
||||
uint32_t subdeviceId = 0;
|
||||
uint32_t powerLimitCount = 0;
|
||||
class PowerLimitRestorer;
|
||||
|
||||
ze_result_t getErrorCode(ze_result_t result) {
|
||||
if (result == ZE_RESULT_ERROR_NOT_AVAILABLE) {
|
||||
@@ -65,6 +62,10 @@ class LinuxPowerImp : public OsPower, NEO::NonCopyableOrMovableClass {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
ze_result_t getMinLimit(int32_t &minLimit);
|
||||
ze_result_t getMaxLimit(int32_t &maxLimit);
|
||||
ze_result_t getDefaultLimit(int32_t &defaultLimit);
|
||||
};
|
||||
} // namespace Sysman
|
||||
} // namespace L0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (C) 2020-2023 Intel Corporation
|
||||
* Copyright (C) 2020-2024 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
@@ -40,7 +40,7 @@ static ze_result_t readSchedulerValueFromSysfs(SysfsName schedulerSysfsName,
|
||||
}
|
||||
result = pSysmanImp->getSysfsAccess().read(path, readValue);
|
||||
if (result == ZE_RESULT_SUCCESS) {
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(SysmanKmdInterface::microSecond,
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(SysmanKmdInterface::micro,
|
||||
pSysmanKmdInterface->getNativeUnit(schedulerSysfsName),
|
||||
readValue, readValue);
|
||||
readValueVec[i] = readValue;
|
||||
@@ -79,7 +79,7 @@ static ze_result_t writeSchedulerValueToSysfs(SysfsName schedulerSysfsName,
|
||||
auto pSysmanKmdInterface = pSysmanImp->getSysmanKmdInterface();
|
||||
auto sysfsName = pSysmanKmdInterface->getSysfsFilePath(schedulerSysfsName, subdeviceId, false);
|
||||
pSysmanKmdInterface->convertSysfsValueUnit(pSysmanKmdInterface->getNativeUnit(schedulerSysfsName),
|
||||
SysmanKmdInterface::microSecond, writeValue, writeValue);
|
||||
SysmanKmdInterface::micro, writeValue, writeValue);
|
||||
auto engineBasePath = pSysmanKmdInterface->getEngineBasePath(subdeviceId);
|
||||
for (const auto &engineName : listOfEngines) {
|
||||
auto path = engineBasePath + "/" + engineName + "/" + sysfsName;
|
||||
|
||||
@@ -133,9 +133,9 @@ void SysmanKmdInterface::convertSysfsValueUnit(const SysfsValueUnit dstUnit, con
|
||||
dstValue = srcValue;
|
||||
|
||||
if (dstUnit != srcUnit) {
|
||||
if (dstUnit == SysfsValueUnit::milliSecond && srcUnit == SysfsValueUnit::microSecond) {
|
||||
if (dstUnit == SysfsValueUnit::milli && srcUnit == SysfsValueUnit::micro) {
|
||||
dstValue = srcValue / 1000u;
|
||||
} else if (dstUnit == SysfsValueUnit::microSecond && srcUnit == SysfsValueUnit::milliSecond) {
|
||||
} else if (dstUnit == SysfsValueUnit::micro && srcUnit == SysfsValueUnit::milli) {
|
||||
dstValue = srcValue * 1000u;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,8 +99,8 @@ class SysmanKmdInterface {
|
||||
SysmanKmdInterface();
|
||||
virtual ~SysmanKmdInterface();
|
||||
enum SysfsValueUnit : uint32_t {
|
||||
milliSecond,
|
||||
microSecond,
|
||||
milli,
|
||||
micro,
|
||||
unAvailable,
|
||||
};
|
||||
static std::unique_ptr<SysmanKmdInterface> create(NEO::Drm &drm);
|
||||
@@ -195,9 +195,12 @@ class SysmanKmdInterfaceI915Upstream : public SysmanKmdInterface, SysmanKmdInter
|
||||
return sysfsNameToNativeUnitMap;
|
||||
}
|
||||
const std::map<SysfsName, SysfsValueUnit> sysfsNameToNativeUnitMap = {
|
||||
{SysfsName::sysfsNameSchedulerTimeout, milliSecond},
|
||||
{SysfsName::sysfsNameSchedulerTimeslice, milliSecond},
|
||||
{SysfsName::sysfsNameSchedulerWatchDogTimeout, milliSecond},
|
||||
{SysfsName::sysfsNameSchedulerTimeout, milli},
|
||||
{SysfsName::sysfsNameSchedulerTimeslice, milli},
|
||||
{SysfsName::sysfsNameSchedulerWatchDogTimeout, milli},
|
||||
{SysfsName::sysfsNameSustainedPowerLimit, micro},
|
||||
{SysfsName::sysfsNameCriticalPowerLimit, micro},
|
||||
{SysfsName::sysfsNameDefaultPowerLimit, micro},
|
||||
};
|
||||
};
|
||||
|
||||
@@ -238,9 +241,12 @@ class SysmanKmdInterfaceI915Prelim : public SysmanKmdInterface, SysmanKmdInterfa
|
||||
return sysfsNameToNativeUnitMap;
|
||||
}
|
||||
const std::map<SysfsName, SysfsValueUnit> sysfsNameToNativeUnitMap = {
|
||||
{SysfsName::sysfsNameSchedulerTimeout, milliSecond},
|
||||
{SysfsName::sysfsNameSchedulerTimeslice, milliSecond},
|
||||
{SysfsName::sysfsNameSchedulerWatchDogTimeout, milliSecond},
|
||||
{SysfsName::sysfsNameSchedulerTimeout, milli},
|
||||
{SysfsName::sysfsNameSchedulerTimeslice, milli},
|
||||
{SysfsName::sysfsNameSchedulerWatchDogTimeout, milli},
|
||||
{SysfsName::sysfsNameSustainedPowerLimit, micro},
|
||||
{SysfsName::sysfsNameCriticalPowerLimit, micro},
|
||||
{SysfsName::sysfsNameDefaultPowerLimit, micro},
|
||||
};
|
||||
};
|
||||
|
||||
@@ -283,10 +289,13 @@ class SysmanKmdInterfaceXe : public SysmanKmdInterface {
|
||||
return sysfsNameToNativeUnitMap;
|
||||
}
|
||||
const std::map<SysfsName, SysfsValueUnit> sysfsNameToNativeUnitMap = {
|
||||
{SysfsName::sysfsNameSchedulerTimeout, microSecond},
|
||||
{SysfsName::sysfsNameSchedulerTimeslice, microSecond},
|
||||
{SysfsName::sysfsNameSchedulerWatchDogTimeout, milliSecond},
|
||||
{SysfsName::sysfsNameSchedulerWatchDogTimeoutMaximum, milliSecond},
|
||||
{SysfsName::sysfsNameSchedulerTimeout, micro},
|
||||
{SysfsName::sysfsNameSchedulerTimeslice, micro},
|
||||
{SysfsName::sysfsNameSchedulerWatchDogTimeout, milli},
|
||||
{SysfsName::sysfsNameSchedulerWatchDogTimeoutMaximum, milli},
|
||||
{SysfsName::sysfsNameSustainedPowerLimit, micro},
|
||||
{SysfsName::sysfsNameCriticalPowerLimit, micro},
|
||||
{SysfsName::sysfsNameDefaultPowerLimit, micro},
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user