Revert "feature(debugger): online page fault event handling"

This reverts commit 0c6444aab7.

Signed-off-by: Compute-Runtime-Validation <compute-runtime-validation@intel.com>
This commit is contained in:
Compute-Runtime-Validation
2023-08-24 03:39:37 +02:00
committed by Compute-Runtime-Automation
parent d1f096ad14
commit e61c5af7ec
12 changed files with 60 additions and 485 deletions

View File

@@ -821,12 +821,7 @@ void DebugSessionImp::fillResumeAndStoppedThreadsFromNewlyStopped(std::vector<Eu
memset(reg.get(), 0, regSize); memset(reg.get(), 0, regSize);
readRegistersImp(newlyStopped, ZET_DEBUG_REGSET_TYPE_CR_INTEL_GPU, 0, 1, reg.get()); readRegistersImp(newlyStopped, ZET_DEBUG_REGSET_TYPE_CR_INTEL_GPU, 0, 1, reg.get());
if (allThreads[newlyStopped]->getPageFault()) { if (isForceExceptionOrForceExternalHaltOnlyExceptionReason(reg.get())) {
const uint32_t cr0PFBit16 = 0x10000;
reg[1] = reg[1] | cr0PFBit16;
writeRegistersImp(newlyStopped, ZET_DEBUG_REGSET_TYPE_CR_INTEL_GPU, 0, 1, reg.get());
}
if (isForceExceptionOrForceExternalHaltOnlyExceptionReason(reg.get()) && !allThreads[newlyStopped]->getPageFault()) {
bool threadWasInterrupted = false; bool threadWasInterrupted = false;
for (auto &request : pendingInterrupts) { for (auto &request : pendingInterrupts) {
@@ -850,7 +845,6 @@ void DebugSessionImp::fillResumeAndStoppedThreadsFromNewlyStopped(std::vector<Eu
} }
} else { } else {
PRINT_DEBUGGER_THREAD_LOG("Newly stopped thread = %s, exception bits = %#010" PRIx32 "\n", allThreads[newlyStopped]->toString().c_str(), reg[1]); PRINT_DEBUGGER_THREAD_LOG("Newly stopped thread = %s, exception bits = %#010" PRIx32 "\n", allThreads[newlyStopped]->toString().c_str(), reg[1]);
allThreads[newlyStopped]->setPageFault(false);
stoppedThreadsToReport.push_back(newlyStopped); stoppedThreadsToReport.push_back(newlyStopped);
} }
} }

View File

@@ -163,13 +163,6 @@ class EuThread {
DEBUG_BREAK_IF(reportedAsStopped && state != State::Stopped); DEBUG_BREAK_IF(reportedAsStopped && state != State::Stopped);
return reportedAsStopped; return reportedAsStopped;
} }
void setPageFault(bool value) {
hasPageFault = value;
}
bool getPageFault() {
return hasPageFault;
;
}
public: public:
static constexpr uint64_t invalidHandle = std::numeric_limits<uint64_t>::max(); static constexpr uint64_t invalidHandle = std::numeric_limits<uint64_t>::max();
@@ -180,7 +173,6 @@ class EuThread {
uint8_t systemRoutineCounter = 0; uint8_t systemRoutineCounter = 0;
std::atomic<uint64_t> memoryHandle = invalidHandle; std::atomic<uint64_t> memoryHandle = invalidHandle;
std::atomic<bool> reportedAsStopped = false; std::atomic<bool> reportedAsStopped = false;
bool hasPageFault = false;
}; };
static_assert(sizeof(EuThread::ThreadId) == sizeof(uint64_t)); static_assert(sizeof(EuThread::ThreadId) == sizeof(uint64_t));

View File

@@ -718,6 +718,7 @@ void DebugSessionLinux::handleEvent(prelim_drm_i915_debug_event *event) {
(int)attention->base.flags, (uint64_t)attention->base.seqno, (uint64_t)attention->base.size, (int)attention->base.flags, (uint64_t)attention->base.seqno, (uint64_t)attention->base.size,
(uint64_t)attention->client_handle, (uint64_t)attention->flags, (uint32_t)attention->ci.engine_class, (uint64_t)attention->client_handle, (uint64_t)attention->flags, (uint32_t)attention->ci.engine_class,
(uint32_t)attention->ci.engine_instance, (uint32_t)attention->bitmask_size, uint64_t(attention->ctx_handle)); (uint32_t)attention->ci.engine_instance, (uint32_t)attention->bitmask_size, uint64_t(attention->ctx_handle));
handleAttentionEvent(attention); handleAttentionEvent(attention);
} break; } break;
@@ -725,17 +726,6 @@ void DebugSessionLinux::handleEvent(prelim_drm_i915_debug_event *event) {
prelim_drm_i915_debug_event_engines *engines = reinterpret_cast<prelim_drm_i915_debug_event_engines *>(event); prelim_drm_i915_debug_event_engines *engines = reinterpret_cast<prelim_drm_i915_debug_event_engines *>(event);
handleEnginesEvent(engines); handleEnginesEvent(engines);
} break; } break;
case PRELIM_DRM_I915_DEBUG_EVENT_PAGE_FAULT: {
prelim_drm_i915_debug_event_page_fault *pf = reinterpret_cast<prelim_drm_i915_debug_event_page_fault *>(event);
PRINT_DEBUGGER_INFO_LOG("PRELIM_I915_DEBUG_IOCTL_READ_EVENT type: PRELIM_DRM_I915_DEBUG_EVENT_PAGE_FAULT flags = %d, address = %llu seqno = %d, size = %llu"
" client_handle = %llu flags = %llu class = %lu instance = %lu bitmask_size = %lu ctx_handle = %llu\n",
(int)pf->base.flags, (uint64_t)pf->page_fault_address, (uint64_t)pf->base.seqno, (uint64_t)pf->base.size,
(uint64_t)pf->client_handle, (uint64_t)pf->flags, (uint32_t)pf->ci.engine_class,
(uint32_t)pf->ci.engine_instance, (uint32_t)pf->bitmask_size, uint64_t(pf->ctx_handle));
handlePageFaultEvent(pf);
} break;
default: default:
PRINT_DEBUGGER_INFO_LOG("PRELIM_I915_DEBUG_IOCTL_READ_EVENT type: UNHANDLED %d flags = %d size = %llu\n", (int)event->type, (int)event->flags, (uint64_t)event->size); PRINT_DEBUGGER_INFO_LOG("PRELIM_I915_DEBUG_IOCTL_READ_EVENT type: UNHANDLED %d flags = %d size = %llu\n", (int)event->type, (int)event->flags, (uint64_t)event->size);
break; break;
@@ -815,14 +805,20 @@ void DebugSessionLinux::readStateSaveAreaHeader() {
ze_result_t DebugSessionLinux::readEventImp(prelim_drm_i915_debug_event *drmDebugEvent) { ze_result_t DebugSessionLinux::readEventImp(prelim_drm_i915_debug_event *drmDebugEvent) {
auto ret = ioctl(PRELIM_I915_DEBUG_IOCTL_READ_EVENT, drmDebugEvent); auto ret = ioctl(PRELIM_I915_DEBUG_IOCTL_READ_EVENT, drmDebugEvent);
if (ret != 0) { if (ret != 0) {
PRINT_DEBUGGER_ERROR_LOG("PRELIM_I915_DEBUG_IOCTL_READ_EVENT failed: retCode: %d errno = %d\n", ret, errno); PRINT_DEBUGGER_ERROR_LOG("PRELIM_I915_DEBUG_IOCTL_READ_EVENT failed: retCode: %d errno = %d\n", ret, errno);
return ZE_RESULT_NOT_READY; } else {
} else if (drmDebugEvent->flags & ~static_cast<uint32_t>(PRELIM_DRM_I915_DEBUG_EVENT_CREATE | PRELIM_DRM_I915_DEBUG_EVENT_DESTROY | PRELIM_DRM_I915_DEBUG_EVENT_STATE_CHANGE | PRELIM_DRM_I915_DEBUG_EVENT_NEED_ACK)) { if ((drmDebugEvent->flags & PRELIM_DRM_I915_DEBUG_EVENT_CREATE) == 0 &&
PRINT_DEBUGGER_ERROR_LOG("PRELIM_I915_DEBUG_IOCTL_READ_EVENT unsupported flag = %d\n", (int)drmDebugEvent->flags); (drmDebugEvent->flags & PRELIM_DRM_I915_DEBUG_EVENT_DESTROY) == 0 &&
return ZE_RESULT_ERROR_UNKNOWN; (drmDebugEvent->flags & PRELIM_DRM_I915_DEBUG_EVENT_STATE_CHANGE) == 0) {
PRINT_DEBUGGER_ERROR_LOG("PRELIM_I915_DEBUG_IOCTL_READ_EVENT unsupported flag = %d\n", (int)drmDebugEvent->flags);
return ZE_RESULT_ERROR_UNKNOWN;
}
return ZE_RESULT_SUCCESS;
} }
return ZE_RESULT_SUCCESS; return ZE_RESULT_NOT_READY;
} }
bool DebugSessionLinux::handleVmBindEvent(prelim_drm_i915_debug_event_vm_bind *vmBind) { bool DebugSessionLinux::handleVmBindEvent(prelim_drm_i915_debug_event_vm_bind *vmBind) {
@@ -1258,25 +1254,6 @@ void DebugSessionLinux::handleContextParamEvent(prelim_drm_i915_debug_event_cont
} }
} }
uint64_t DebugSessionLinux::getVmHandleFromClientAndlrcHandle(uint64_t clientHandle, uint64_t lrcHandle) {
if (clientHandleToConnection.find(clientHandle) == clientHandleToConnection.end()) {
return invalidHandle;
}
auto &clientConnection = clientHandleToConnection[clientHandle];
if (clientConnection->lrcToContextHandle.find(lrcHandle) == clientConnection->lrcToContextHandle.end()) {
return invalidHandle;
}
auto contextHandle = clientConnection->lrcToContextHandle[lrcHandle];
if (clientConnection->contextsCreated.find(contextHandle) == clientConnection->contextsCreated.end()) {
return invalidHandle;
}
return clientConnection->contextsCreated[contextHandle].vm;
}
void DebugSessionLinux::handleAttentionEvent(prelim_drm_i915_debug_event_eu_attention *attention) { void DebugSessionLinux::handleAttentionEvent(prelim_drm_i915_debug_event_eu_attention *attention) {
NEO::EngineClassInstance engineClassInstance = {attention->ci.engine_class, attention->ci.engine_instance}; NEO::EngineClassInstance engineClassInstance = {attention->ci.engine_class, attention->ci.engine_instance};
auto tileIndex = DrmHelper::getEngineTileIndex(connectedDevice, engineClassInstance); auto tileIndex = DrmHelper::getEngineTileIndex(connectedDevice, engineClassInstance);
@@ -1287,10 +1264,23 @@ void DebugSessionLinux::handleAttentionEvent(prelim_drm_i915_debug_event_eu_atte
return; return;
} }
newAttentionRaised( newAttentionRaised(tileIndex);
tileIndex);
auto vmHandle = getVmHandleFromClientAndlrcHandle(attention->client_handle, attention->lrc_handle); if (clientHandleToConnection.find(attention->client_handle) == clientHandleToConnection.end()) {
return;
}
auto &clientConnection = clientHandleToConnection[attention->client_handle];
if (clientConnection->lrcToContextHandle.find(attention->lrc_handle) == clientConnection->lrcToContextHandle.end()) {
return;
}
auto contextHandle = clientConnection->lrcToContextHandle[attention->lrc_handle];
if (clientConnection->contextsCreated.find(contextHandle) == clientConnection->contextsCreated.end()) {
return;
}
auto vmHandle = clientConnection->contextsCreated[contextHandle].vm;
if (vmHandle == invalidHandle) { if (vmHandle == invalidHandle) {
return; return;
} }
@@ -1366,78 +1356,6 @@ void DebugSessionLinux::handleAttentionEvent(prelim_drm_i915_debug_event_eu_atte
} }
} }
void DebugSessionLinux::handlePageFaultEvent(prelim_drm_i915_debug_event_page_fault *pf) {
NEO::EngineClassInstance engineClassInstance = {pf->ci.engine_class, pf->ci.engine_instance};
auto tileIndex = DrmHelper::getEngineTileIndex(connectedDevice, engineClassInstance);
DEBUG_BREAK_IF(pf->bitmask_size % 3u != 0u);
size_t size = pf->bitmask_size / 3;
uint8_t *bitmaskBefore = &pf->bitmask[0];
uint8_t *bitmaskAfter = &pf->bitmask[size];
uint8_t *bitmaskResolved = &pf->bitmask[size * 2];
PRINT_DEBUGGER_INFO_LOG("PageFault event BEFORE", 0);
printBitmask(bitmaskBefore, size);
PRINT_DEBUGGER_INFO_LOG("PageFault event AFTER", 0);
printBitmask(bitmaskAfter, size);
PRINT_DEBUGGER_INFO_LOG("PageFault event RESOLVED", 0);
printBitmask(bitmaskResolved, size);
auto vmHandle = getVmHandleFromClientAndlrcHandle(pf->client_handle, pf->lrc_handle);
if (vmHandle == invalidHandle) {
return;
}
if (!connectedDevice->getNEODevice()->getDeviceBitfield().test(tileIndex)) {
return;
}
std::unique_ptr<uint8_t[]> bitmaskPF = std::make_unique<uint8_t[]>(size);
std::transform(bitmaskAfter, bitmaskAfter + size, bitmaskResolved, bitmaskPF.get(), std::bit_xor<uint8_t>());
auto hwInfo = connectedDevice->getHwInfo();
auto &l0GfxCoreHelper = connectedDevice->getL0GfxCoreHelper();
auto threadsWithPF = l0GfxCoreHelper.getThreadsFromAttentionBitmask(hwInfo, tileIndex, bitmaskPF.get(), size);
auto stoppedThreads = l0GfxCoreHelper.getThreadsFromAttentionBitmask(hwInfo, tileIndex, bitmaskResolved, size);
if (threadsWithPF.size() == 0) {
zet_debug_event_t debugEvent = {};
debugEvent.type = ZET_DEBUG_EVENT_TYPE_PAGE_FAULT;
PRINT_DEBUGGER_INFO_LOG("PageFault event for unknown thread", 0);
enqueueApiEvent(debugEvent);
}
auto gpuVa = getContextStateSaveAreaGpuVa(vmHandle);
auto stateSaveAreaSize = getContextStateSaveAreaSize(vmHandle);
allocateStateSaveAreaMemory(stateSaveAreaSize);
auto stateSaveReadResult = readGpuMemory(vmHandle, stateSaveAreaMemory.data(), stateSaveAreaSize, gpuVa);
if (stateSaveReadResult == ZE_RESULT_SUCCESS) {
std::unique_lock<std::mutex> lock;
if (tileSessionsEnabled) {
lock = std::unique_lock<std::mutex>(static_cast<TileDebugSessionLinux *>(tileSessions[tileIndex].first)->threadStateMutex);
} else {
lock = std::unique_lock<std::mutex>(threadStateMutex);
}
for (auto threadId : threadsWithPF) {
PRINT_DEBUGGER_INFO_LOG("PageFault event for thread %s", EuThread::toString(threadId).c_str());
allThreads[threadId]->setPageFault(true);
}
for (auto threadId : stoppedThreads) {
if (tileSessionsEnabled) {
static_cast<TileDebugSessionLinux *>(tileSessions[tileIndex].first)->addThreadToNewlyStoppedFromRaisedAttention(threadId, vmHandle, stateSaveAreaMemory.data());
} else {
addThreadToNewlyStoppedFromRaisedAttention(threadId, vmHandle, stateSaveAreaMemory.data());
}
}
}
if (tileSessionsEnabled) {
static_cast<TileDebugSessionLinux *>(tileSessions[tileIndex].first)->checkTriggerEventsForAttention();
} else {
checkTriggerEventsForAttention();
}
return;
}
void DebugSessionLinux::handleEnginesEvent(prelim_drm_i915_debug_event_engines *engines) { void DebugSessionLinux::handleEnginesEvent(prelim_drm_i915_debug_event_engines *engines) {
PRINT_DEBUGGER_INFO_LOG("ENGINES event: client_handle = %llu, ctx_handle = %llu, num_engines = %llu %s\n", PRINT_DEBUGGER_INFO_LOG("ENGINES event: client_handle = %llu, ctx_handle = %llu, num_engines = %llu %s\n",
(uint64_t)engines->client_handle, (uint64_t)engines->client_handle,

View File

@@ -245,12 +245,10 @@ struct DebugSessionLinux : DebugSessionImp {
return 0.5; return 0.5;
} }
uint64_t getVmHandleFromClientAndlrcHandle(uint64_t clientHandle, uint64_t lrcHandle);
bool handleVmBindEvent(prelim_drm_i915_debug_event_vm_bind *vmBind); bool handleVmBindEvent(prelim_drm_i915_debug_event_vm_bind *vmBind);
void handleContextParamEvent(prelim_drm_i915_debug_event_context_param *contextParam); void handleContextParamEvent(prelim_drm_i915_debug_event_context_param *contextParam);
void handleAttentionEvent(prelim_drm_i915_debug_event_eu_attention *attention); void handleAttentionEvent(prelim_drm_i915_debug_event_eu_attention *attention);
void handleEnginesEvent(prelim_drm_i915_debug_event_engines *engines); void handleEnginesEvent(prelim_drm_i915_debug_event_engines *engines);
void handlePageFaultEvent(prelim_drm_i915_debug_event_page_fault *pf);
virtual bool ackIsaEvents(uint32_t deviceIndex, uint64_t isaVa); virtual bool ackIsaEvents(uint32_t deviceIndex, uint64_t isaVa);
virtual bool ackModuleEvents(uint32_t deviceIndex, uint64_t moduleUuidHandle); virtual bool ackModuleEvents(uint32_t deviceIndex, uint64_t moduleUuidHandle);

View File

@@ -756,45 +756,6 @@ TEST(DebugSessionTest, givenStoppedThreadsWhenFillingResumeAndStoppedThreadsFrom
} }
} }
TEST(DebugSessionTest, givenThreadsStoppedWithPageFaultWhenCallingfillResumeAndStoppedThreadsFromNewlyStoppedThenCRIsWritten) {
zet_debug_config_t config = {};
config.pid = 0x1234;
auto hwInfo = *NEO::defaultHwInfo.get();
NEO::MockDevice *neoDevice(NEO::MockDevice::createWithNewExecutionEnvironment<NEO::MockDevice>(&hwInfo, 0));
Mock<L0::DeviceImp> deviceImp(neoDevice, neoDevice->getExecutionEnvironment());
auto sessionMock = std::make_unique<MockDebugSession>(config, &deviceImp);
EuThread::ThreadId thread = {0, 0, 0, 0, 1};
sessionMock->newlyStoppedThreads.push_back(thread);
sessionMock->onlyForceException = true;
std::vector<EuThread::ThreadId> resumeThreads;
std::vector<EuThread::ThreadId> stoppedThreads;
std::vector<EuThread::ThreadId> interruptedThreads;
sessionMock->allThreads[thread]->stopThread(1u);
sessionMock->allThreads[thread]->setPageFault(true);
sessionMock->fillResumeAndStoppedThreadsFromNewlyStopped(resumeThreads, stoppedThreads, interruptedThreads);
EXPECT_EQ(0u, resumeThreads.size());
EXPECT_EQ(1u, stoppedThreads.size());
EXPECT_EQ(1u, sessionMock->writeRegistersCallCount);
EXPECT_EQ(ZET_DEBUG_REGSET_TYPE_CR_INTEL_GPU, sessionMock->writeRegistersReg);
EXPECT_EQ(false, sessionMock->allThreads[thread]->getPageFault());
resumeThreads.clear();
stoppedThreads.clear();
sessionMock->newlyStoppedThreads.push_back(thread);
sessionMock->onlyForceException = true;
sessionMock->fillResumeAndStoppedThreadsFromNewlyStopped(resumeThreads, stoppedThreads, interruptedThreads);
EXPECT_EQ(1u, resumeThreads.size());
EXPECT_EQ(0u, stoppedThreads.size());
EXPECT_EQ(1u, sessionMock->writeRegistersCallCount);
}
TEST(DebugSessionTest, givenNoThreadsStoppedWhenCallingfillResumeAndStoppedThreadsFromNewlyStoppedThenReadStateSaveAreaNotCalled) { TEST(DebugSessionTest, givenNoThreadsStoppedWhenCallingfillResumeAndStoppedThreadsFromNewlyStoppedThenReadStateSaveAreaNotCalled) {
zet_debug_config_t config = {}; zet_debug_config_t config = {};
config.pid = 0x1234; config.pid = 0x1234;

View File

@@ -255,17 +255,5 @@ TEST(EuThread, GivenEuThreadWhenGettingLastCounterThenCorrectValueIsReturned) {
EXPECT_EQ(9u, euThread.getLastCounter()); EXPECT_EQ(9u, euThread.getLastCounter());
} }
TEST(EuThread, GivenEuThreadWhenGettingPageFaultThenCorrectValueIsReturned) {
ze_device_thread_t devThread = {3, 4, 5, 6};
EuThread::ThreadId threadId(0, devThread);
EuThread euThread(threadId);
EXPECT_EQ(false, euThread.getPageFault());
euThread.setPageFault(true);
EXPECT_EQ(true, euThread.getPageFault());
euThread.setPageFault(false);
EXPECT_EQ(false, euThread.getPageFault());
}
} // namespace ult } // namespace ult
} // namespace L0 } // namespace L0

View File

@@ -20,7 +20,6 @@
#include "level_zero/core/test/unit_tests/fixtures/device_fixture.h" #include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
#include "level_zero/core/test/unit_tests/mocks/mock_built_ins.h" #include "level_zero/core/test/unit_tests/mocks/mock_built_ins.h"
#include "level_zero/tools/source/debug/linux/prelim/debug_session.h" #include "level_zero/tools/source/debug/linux/prelim/debug_session.h"
#include "level_zero/tools/test/unit_tests/sources/debug/debug_session_common.h"
#include "common/StateSaveAreaHeader.h" #include "common/StateSaveAreaHeader.h"
@@ -620,61 +619,6 @@ struct DebugApiLinuxFixture : public DeviceFixture {
static constexpr uint8_t bufferSize = 16; static constexpr uint8_t bufferSize = 16;
}; };
struct DebugApiPageFaultEventFixture : public DebugApiLinuxFixture {
void setUp() {
DebugApiLinuxFixture::setUp();
zet_debug_config_t config = {};
config.pid = 0x1234;
sessionMock = std::make_unique<MockDebugSessionLinux>(config, device, 10);
ASSERT_NE(nullptr, sessionMock);
sessionMock->clientHandle = MockDebugSessionLinux::mockClientHandle;
auto handler = new MockIoctlHandler;
sessionMock->ioctlHandler.reset(handler);
SIP::version version = {2, 0, 0};
initStateSaveArea(sessionMock->stateSaveAreaHeader, version, device);
handler->setPreadMemory(sessionMock->stateSaveAreaHeader.data(), sessionMock->stateSaveAreaHeader.size(), 0x1000);
sessionMock->clientHandleToConnection[MockDebugSessionLinux::mockClientHandle]->contextsCreated[ctxHandle].vm = vmHandle;
sessionMock->clientHandleToConnection[MockDebugSessionLinux::mockClientHandle]->lrcToContextHandle[lrcHandle] = ctxHandle;
DebugSessionLinux::BindInfo cssaInfo = {0x1000, sessionMock->stateSaveAreaHeader.size()};
sessionMock->clientHandleToConnection[MockDebugSessionLinux::mockClientHandle]->vmToContextStateSaveAreaBindInfo[vmHandle] = cssaInfo;
}
void tearDown() {
DebugApiLinuxFixture::tearDown();
}
void buildPfi915Event() {
buildPfi915Event(MockDebugSessionLinux::mockClientHandle);
}
void buildPfi915Event(uint64_t clientHandle) {
prelim_drm_i915_debug_event_page_fault pf = {};
pf.base.type = PRELIM_DRM_I915_DEBUG_EVENT_PAGE_FAULT;
pf.base.flags = 0;
pf.base.size = sizeof(prelim_drm_i915_debug_event_page_fault) + (bitmaskSize * 3u);
pf.client_handle = clientHandle;
pf.lrc_handle = lrcHandle;
pf.flags = 0;
pf.ci.engine_class = 0;
pf.ci.engine_instance = 0;
pf.bitmask_size = static_cast<uint32_t>(bitmaskSize * 3);
memcpy(data, &pf, sizeof(prelim_drm_i915_debug_event_page_fault));
memcpy(ptrOffset(data, offsetof(prelim_drm_i915_debug_event_page_fault, bitmask)), bitmaskBefore.get(), bitmaskSize);
memcpy(ptrOffset(data, offsetof(prelim_drm_i915_debug_event_page_fault, bitmask) + bitmaskSize), bitmaskAfter.get(), bitmaskSize);
memcpy(ptrOffset(data, offsetof(prelim_drm_i915_debug_event_page_fault, bitmask) + (2 * bitmaskSize)), bitmaskResolved.get(), bitmaskSize);
}
size_t bitmaskSize = 256;
uint8_t data[sizeof(prelim_drm_i915_debug_event_page_fault) + (256 * 3)];
std::unique_ptr<uint8_t[]> bitmaskBefore, bitmaskAfter, bitmaskResolved;
std::unique_ptr<MockDebugSessionLinux> sessionMock;
uint64_t ctxHandle = 2;
uint64_t vmHandle = 7;
uint64_t lrcHandle = 8;
};
struct DebugApiLinuxMultiDeviceFixture : public MultipleDevicesWithCustomHwInfo { struct DebugApiLinuxMultiDeviceFixture : public MultipleDevicesWithCustomHwInfo {
void setUp(); void setUp();

View File

@@ -3044,6 +3044,7 @@ TEST_F(DebugApiLinuxTest, GivenDebugSessionWhenClientCreateAndDestroyEventsReadO
EXPECT_EQ(ZE_RESULT_ERROR_DEVICE_LOST, result); EXPECT_EQ(ZE_RESULT_ERROR_DEVICE_LOST, result);
EXPECT_EQ(eventsCount, static_cast<size_t>(session->getInternalEventCounter.load())); EXPECT_EQ(eventsCount, static_cast<size_t>(session->getInternalEventCounter.load()));
} }
TEST_F(DebugApiLinuxTest, GivenEventWithInvalidFlagsWhenReadingEventThenUnknownErrorIsReturned) { TEST_F(DebugApiLinuxTest, GivenEventWithInvalidFlagsWhenReadingEventThenUnknownErrorIsReturned) {
zet_debug_config_t config = {}; zet_debug_config_t config = {};
config.pid = 0x1234; config.pid = 0x1234;
@@ -3067,7 +3068,7 @@ TEST_F(DebugApiLinuxTest, GivenEventWithInvalidFlagsWhenReadingEventThenUnknownE
auto memory = std::make_unique<uint64_t[]>(MockDebugSessionLinux::maxEventSize / sizeof(uint64_t)); auto memory = std::make_unique<uint64_t[]>(MockDebugSessionLinux::maxEventSize / sizeof(uint64_t));
prelim_drm_i915_debug_event *event = reinterpret_cast<prelim_drm_i915_debug_event *>(memory.get()); prelim_drm_i915_debug_event *event = reinterpret_cast<prelim_drm_i915_debug_event *>(memory.get());
event->type = PRELIM_DRM_I915_DEBUG_EVENT_READ; event->type = PRELIM_DRM_I915_DEBUG_EVENT_READ;
event->flags = 0x8000; event->flags = 0;
event->size = MockDebugSessionLinux::maxEventSize; event->size = MockDebugSessionLinux::maxEventSize;
ze_result_t result = session->readEventImp(event); ze_result_t result = session->readEventImp(event);
@@ -3098,6 +3099,33 @@ TEST_F(DebugApiLinuxTest, GivenDebugSessionInitializationWhenNoValidEventsAreRea
EXPECT_EQ(ZE_RESULT_NOT_READY, result); EXPECT_EQ(ZE_RESULT_NOT_READY, result);
} }
TEST_F(DebugApiLinuxTest, GivenInvalidFlagsWhenReadingEventThenUnknownErrorIsReturned) {
zet_debug_config_t config = {};
config.pid = 0x1234;
auto session = std::make_unique<MockDebugSessionLinux>(config, device, 10);
ASSERT_NE(nullptr, session);
prelim_drm_i915_debug_event_client clientInvalidFlag = {};
clientInvalidFlag.base.type = PRELIM_DRM_I915_DEBUG_EVENT_CLIENT;
clientInvalidFlag.base.flags = 0x8000;
clientInvalidFlag.base.size = sizeof(prelim_drm_i915_debug_event_client);
clientInvalidFlag.handle = 1;
auto handler = new MockIoctlHandler;
handler->eventQueue.push({reinterpret_cast<char *>(&clientInvalidFlag), static_cast<uint64_t>(clientInvalidFlag.base.size)});
handler->pollRetVal = 1;
session->ioctlHandler.reset(handler);
uint64_t data[512];
auto drmDebugEvent = reinterpret_cast<prelim_drm_i915_debug_event *>(data);
ze_result_t result = session->readEventImp(drmDebugEvent);
EXPECT_EQ(ZE_RESULT_ERROR_UNKNOWN, result);
EXPECT_EQ(1u, static_cast<size_t>(handler->ioctlCalled));
}
TEST_F(DebugApiLinuxTest, GivenValidFlagsWhenReadingEventThenEventIsNotProcessed) { TEST_F(DebugApiLinuxTest, GivenValidFlagsWhenReadingEventThenEventIsNotProcessed) {
zet_debug_config_t config = {}; zet_debug_config_t config = {};
config.pid = 0x1234; config.pid = 0x1234;
@@ -5843,130 +5871,6 @@ TEST_F(DebugApiLinuxTest, givenTileAttachEnabledWhenDeviceDoesNotHaveTilesThenTi
EXPECT_EQ(0u, session->tileSessions.size()); EXPECT_EQ(0u, session->tileSessions.size());
} }
using DebugApiLinuxPageFaultEventTest = Test<DebugApiPageFaultEventFixture>;
TEST_F(DebugApiLinuxPageFaultEventTest, GivenNoPageFaultingThreadWhenHandlingPageFaultEventThenL0ApiEventGenerated) {
auto &hwInfo = neoDevice->getHardwareInfo();
auto &l0GfxCoreHelper = neoDevice->getRootDeviceEnvironment().getHelper<L0GfxCoreHelper>();
std::vector<EuThread::ThreadId> threads{
{0, 0, 0, 0, 0},
{0, 0, 0, 0, 1},
{0, 0, 0, 0, 2},
{0, 0, 0, 0, 3},
{0, 0, 0, 0, 4},
{0, 0, 0, 0, 5},
{0, 0, 0, 0, 6}};
for (auto thread : threads) {
sessionMock->stoppedThreads[thread.packed] = 1;
}
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threads, hwInfo, bitmaskBefore, bitmaskSize);
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threads, hwInfo, bitmaskAfter, bitmaskSize);
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threads, hwInfo, bitmaskResolved, bitmaskSize);
bitmaskSize = std::min(size_t(128), bitmaskSize);
buildPfi915Event();
sessionMock->handleEvent(reinterpret_cast<prelim_drm_i915_debug_event *>(data));
EXPECT_EQ(threads.size(), sessionMock->newlyStoppedThreads.size());
for (auto thread : threads) {
EXPECT_FALSE(sessionMock->allThreads[thread]->getPageFault());
}
ASSERT_EQ(1u, sessionMock->apiEvents.size());
auto event = sessionMock->apiEvents.front();
ASSERT_EQ(event.type, ZET_DEBUG_EVENT_TYPE_PAGE_FAULT);
}
TEST_F(DebugApiLinuxPageFaultEventTest, GivenPageFaultEventWIthInvalidClientHandleThenNoThreadsReportedStopped) {
auto &hwInfo = neoDevice->getHardwareInfo();
auto &l0GfxCoreHelper = neoDevice->getRootDeviceEnvironment().getHelper<L0GfxCoreHelper>();
std::vector<EuThread::ThreadId> threads;
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threads, hwInfo, bitmaskBefore, bitmaskSize);
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threads, hwInfo, bitmaskAfter, bitmaskSize);
threads.push_back({0, 0, 0, 0, 0});
threads.push_back({0, 0, 0, 0, 2});
threads.push_back({0, 0, 0, 0, 3});
threads.push_back({0, 0, 0, 0, 4});
threads.push_back({0, 0, 0, 0, 6});
for (auto thread : threads) {
sessionMock->stoppedThreads[thread.packed] = 1;
}
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threads, hwInfo, bitmaskResolved, bitmaskSize);
bitmaskSize = std::min(size_t(128), bitmaskSize);
buildPfi915Event(MockDebugSessionLinux::invalidClientHandle);
sessionMock->handleEvent(reinterpret_cast<prelim_drm_i915_debug_event *>(data));
EXPECT_EQ(0u, sessionMock->newlyStoppedThreads.size());
}
TEST_F(DebugApiLinuxPageFaultEventTest, GivenPageFaultEventWhenHandlingEventThenThreadsReportedStoppedAndPfSet) {
auto &hwInfo = neoDevice->getHardwareInfo();
auto &l0GfxCoreHelper = neoDevice->getRootDeviceEnvironment().getHelper<L0GfxCoreHelper>();
std::vector<EuThread::ThreadId> threads;
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threads, hwInfo, bitmaskBefore, bitmaskSize);
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threads, hwInfo, bitmaskAfter, bitmaskSize);
threads.push_back({0, 0, 0, 0, 0});
threads.push_back({0, 0, 0, 0, 2});
threads.push_back({0, 0, 0, 0, 3});
threads.push_back({0, 0, 0, 0, 4});
threads.push_back({0, 0, 0, 0, 6});
for (auto thread : threads) {
sessionMock->stoppedThreads[thread.packed] = 1;
}
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threads, hwInfo, bitmaskResolved, bitmaskSize);
bitmaskSize = std::min(size_t(128), bitmaskSize);
buildPfi915Event();
sessionMock->handleEvent(reinterpret_cast<prelim_drm_i915_debug_event *>(data));
EXPECT_EQ(threads.size(), sessionMock->newlyStoppedThreads.size());
for (auto thread : threads) {
EXPECT_TRUE(sessionMock->allThreads[thread]->getPageFault());
}
}
TEST_F(DebugApiLinuxPageFaultEventTest, GivenPageFaultEventWhenHandlingEventThenThreadsNotNewlyResolvedAreNotMarkedAsPf) {
auto &hwInfo = neoDevice->getHardwareInfo();
auto &l0GfxCoreHelper = neoDevice->getRootDeviceEnvironment().getHelper<L0GfxCoreHelper>();
std::vector<EuThread::ThreadId> threadsBefore, threadsAfter, threadsResolved;
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threadsBefore, hwInfo, bitmaskBefore, bitmaskSize);
threadsAfter.push_back({0, 0, 0, 0, 0});
threadsAfter.push_back({0, 0, 0, 0, 1});
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threadsAfter, hwInfo, bitmaskAfter, bitmaskSize);
threadsResolved.push_back({0, 0, 0, 0, 0});
threadsResolved.push_back({0, 0, 0, 0, 1});
threadsResolved.push_back({0, 0, 0, 0, 2});
threadsResolved.push_back({0, 0, 0, 0, 3});
for (auto thread : threadsResolved) {
sessionMock->stoppedThreads[thread.packed] = 1;
}
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads(threadsResolved, hwInfo, bitmaskResolved, bitmaskSize);
bitmaskSize = std::min(size_t(128), bitmaskSize);
buildPfi915Event();
sessionMock->handleEvent(reinterpret_cast<prelim_drm_i915_debug_event *>(data));
EXPECT_EQ(threadsResolved.size(), sessionMock->newlyStoppedThreads.size());
for (auto thread : threadsResolved) {
if (std::find(threadsAfter.begin(), threadsAfter.end(), thread) == threadsAfter.end()) {
EXPECT_TRUE(sessionMock->allThreads[thread]->getPageFault());
} else {
EXPECT_FALSE(sessionMock->allThreads[thread]->getPageFault());
}
}
}
using DebugApiLinuxAttentionTest = Test<DebugApiLinuxFixture>; using DebugApiLinuxAttentionTest = Test<DebugApiLinuxFixture>;
TEST_F(DebugApiLinuxAttentionTest, GivenEuAttentionEventForThreadsWhenHandlingEventThenNewlyStoppedThreadsSaved) { TEST_F(DebugApiLinuxAttentionTest, GivenEuAttentionEventForThreadsWhenHandlingEventThenNewlyStoppedThreadsSaved) {
@@ -8629,37 +8533,6 @@ TEST_F(AffinityMaskMultipleSubdevicesTestLinux, GivenEventWithAckFlagAndTileNotW
EXPECT_EQ(vmBindIsa->base.seqno, handler->debugEventAcked.seqno); EXPECT_EQ(vmBindIsa->base.seqno, handler->debugEventAcked.seqno);
} }
TEST_F(AffinityMaskMultipleSubdevicesTestLinux, GivenPfEventForTileNotWithinBitfieldWhenHandlingEventThenEventIsSkipped) {
auto debugSession = std::make_unique<MockDebugSessionLinux>(zet_debug_config_t{1234}, deviceImp, 10);
uint64_t ctxHandle = 2;
uint64_t vmHandle = 7;
uint64_t lrcHandle = 8;
debugSession->clientHandleToConnection[debugSession->clientHandle]->contextsCreated[ctxHandle].vm = vmHandle;
debugSession->clientHandleToConnection[debugSession->clientHandle]->lrcToContextHandle[lrcHandle] = ctxHandle;
debugSession->clientHandleToConnection[debugSession->clientHandle]->vmToTile[vmHandle] = 2;
prelim_drm_i915_debug_event_page_fault pf = {};
pf.base.type = PRELIM_DRM_I915_DEBUG_EVENT_PAGE_FAULT;
pf.base.flags = 0;
pf.base.size = sizeof(prelim_drm_i915_debug_event_page_fault);
pf.client_handle = MockDebugSessionLinux::mockClientHandle;
pf.lrc_handle = lrcHandle;
pf.flags = 0;
auto engineInfo = mockDrm->getEngineInfo();
auto ci = engineInfo->getEngineInstance(2, hwInfo.capabilityTable.defaultEngineType);
pf.ci.engine_class = ci->engineClass;
pf.ci.engine_instance = ci->engineInstance;
ze_device_thread_t thread = {0, 0, 0, UINT32_MAX};
debugSession->pendingInterrupts.push_back(std::pair<ze_device_thread_t, bool>(thread, false));
debugSession->handleEvent(&pf.base);
EXPECT_FALSE(debugSession->triggerEvents);
}
TEST_F(AffinityMaskMultipleSubdevicesTestLinux, GivenAttEventForTileNotWithinBitfieldWhenHandlingEventThenEventIsSkipped) { TEST_F(AffinityMaskMultipleSubdevicesTestLinux, GivenAttEventForTileNotWithinBitfieldWhenHandlingEventThenEventIsSkipped) {
auto debugSession = std::make_unique<MockDebugSessionLinux>(zet_debug_config_t{1234}, deviceImp, 10); auto debugSession = std::make_unique<MockDebugSessionLinux>(zet_debug_config_t{1234}, deviceImp, 10);

View File

@@ -819,70 +819,6 @@ TEST_F(TileAttachTest, givenStoppedThreadsWhenHandlingAttentionEventThenStoppedT
EXPECT_TRUE(tileSessions[1]->triggerEvents); EXPECT_TRUE(tileSessions[1]->triggerEvents);
} }
TEST_F(TileAttachTest, givenStoppedThreadsWhenHandlingPageFaultEventThenStoppedThreadsFromEventAreProcessed) {
// debug attach both tiles
rootSession->tileSessions[0].second = true;
rootSession->tileSessions[1].second = true;
uint64_t ctxHandle = 2;
uint64_t vmHandle = 7;
uint64_t lrcHandle = 8;
rootSession->clientHandleToConnection[MockDebugSessionLinux::mockClientHandle]->contextsCreated[ctxHandle].vm = vmHandle;
rootSession->clientHandleToConnection[MockDebugSessionLinux::mockClientHandle]->lrcToContextHandle[lrcHandle] = ctxHandle;
rootSession->clientHandleToConnection[MockDebugSessionLinux::mockClientHandle]->vmToTile[vmHandle] = 1;
SIP::version version = {2, 0, 0};
initStateSaveArea(rootSession->stateSaveAreaHeader, version, deviceImp);
DebugSessionLinux::BindInfo cssaInfo = {reinterpret_cast<uint64_t>(rootSession->stateSaveAreaHeader.data()), rootSession->stateSaveAreaHeader.size()};
rootSession->clientHandleToConnection[MockDebugSessionLinux::mockClientHandle]->vmToContextStateSaveAreaBindInfo[vmHandle] = cssaInfo;
auto handler = new MockIoctlHandler;
rootSession->ioctlHandler.reset(handler);
handler->setPreadMemory(rootSession->stateSaveAreaHeader.data(), rootSession->stateSaveAreaHeader.size(), reinterpret_cast<uint64_t>(rootSession->stateSaveAreaHeader.data()));
uint8_t data[sizeof(prelim_drm_i915_debug_event_page_fault) + 128 * 3];
auto engineInfo = mockDrm->getEngineInfo();
auto engineInstance = engineInfo->getEngineInstance(1, hwInfo.capabilityTable.defaultEngineType);
EuThread::ThreadId thread = {1, 0, 0, 0, 0};
tileSessions[1]->stoppedThreads[thread.packed] = 1;
std::unique_ptr<uint8_t[]> bitmaskBefore, bitmaskAfter, bitmaskResolved;
size_t bitmaskSize = 0;
auto &hwInfo = neoDevice->getHardwareInfo();
auto &l0GfxCoreHelper = neoDevice->getRootDeviceEnvironment().getHelper<L0GfxCoreHelper>();
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads({thread}, hwInfo, bitmaskBefore, bitmaskSize);
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads({thread}, hwInfo, bitmaskAfter, bitmaskSize);
l0GfxCoreHelper.getAttentionBitmaskForSingleThreads({thread}, hwInfo, bitmaskResolved, bitmaskSize);
prelim_drm_i915_debug_event_page_fault pf = {};
pf.base.type = PRELIM_DRM_I915_DEBUG_EVENT_PAGE_FAULT;
pf.base.flags = PRELIM_DRM_I915_DEBUG_EVENT_STATE_CHANGE;
pf.base.size = sizeof(prelim_drm_i915_debug_event_page_fault);
pf.base.seqno = 2;
pf.client_handle = MockDebugSessionLinux::mockClientHandle;
pf.lrc_handle = lrcHandle;
pf.flags = 0;
pf.ci.engine_class = engineInstance->engineClass;
pf.ci.engine_instance = engineInstance->engineInstance;
pf.bitmask_size = static_cast<uint32_t>(bitmaskSize * 3u);
bitmaskSize = std::min(size_t(128), bitmaskSize);
memcpy(data, &pf, sizeof(prelim_drm_i915_debug_event_page_fault));
memcpy(ptrOffset(data, offsetof(prelim_drm_i915_debug_event_page_fault, bitmask)), bitmaskBefore.get(), bitmaskSize);
memcpy(ptrOffset(data, offsetof(prelim_drm_i915_debug_event_page_fault, bitmask) + bitmaskSize), bitmaskAfter.get(), bitmaskSize);
memcpy(ptrOffset(data, offsetof(prelim_drm_i915_debug_event_page_fault, bitmask) + (2 * bitmaskSize)), bitmaskResolved.get(), bitmaskSize);
rootSession->handleEvent(reinterpret_cast<prelim_drm_i915_debug_event *>(data));
auto expectedThreadsToCheck = hwInfo.capabilityTable.fusedEuEnabled ? 2u : 1u;
EXPECT_EQ(expectedThreadsToCheck, tileSessions[1]->newlyStoppedThreads.size());
EXPECT_TRUE(tileSessions[1]->triggerEvents);
}
TEST_F(TileAttachTest, GivenBlockingOnCpuDetachedTileAndZebinModulesWithEventsToAckWhenDetachingTileThenNoAckIoctlIsCalled) { TEST_F(TileAttachTest, GivenBlockingOnCpuDetachedTileAndZebinModulesWithEventsToAckWhenDetachingTileThenNoAckIoctlIsCalled) {
auto handler = new MockIoctlHandler; auto handler = new MockIoctlHandler;
rootSession->ioctlHandler.reset(handler); rootSession->ioctlHandler.reset(handler);

View File

@@ -298,6 +298,7 @@ struct MockDebugSession : public L0::DebugSessionImp {
[[maybe_unused]] auto offset = ptrDiff(gpuVa, reinterpret_cast<uint64_t>(stateSaveAreaHeader.data())); [[maybe_unused]] auto offset = ptrDiff(gpuVa, reinterpret_cast<uint64_t>(stateSaveAreaHeader.data()));
memcpy_s(reinterpret_cast<void *>(gpuVa), size, input, size); memcpy_s(reinterpret_cast<void *>(gpuVa), size, input, size);
} }
return writeMemoryResult; return writeMemoryResult;
} }

View File

@@ -51,7 +51,6 @@ using NEO::PrelimI915::prelim_drm_i915_debug_event_context;
using NEO::PrelimI915::prelim_drm_i915_debug_event_context_param; using NEO::PrelimI915::prelim_drm_i915_debug_event_context_param;
using NEO::PrelimI915::prelim_drm_i915_debug_event_engines; using NEO::PrelimI915::prelim_drm_i915_debug_event_engines;
using NEO::PrelimI915::prelim_drm_i915_debug_event_eu_attention; using NEO::PrelimI915::prelim_drm_i915_debug_event_eu_attention;
using NEO::PrelimI915::prelim_drm_i915_debug_event_page_fault;
using NEO::PrelimI915::prelim_drm_i915_debug_event_uuid; using NEO::PrelimI915::prelim_drm_i915_debug_event_uuid;
using NEO::PrelimI915::prelim_drm_i915_debug_event_vm; using NEO::PrelimI915::prelim_drm_i915_debug_event_vm;
using NEO::PrelimI915::prelim_drm_i915_debug_event_vm_bind; using NEO::PrelimI915::prelim_drm_i915_debug_event_vm_bind;

View File

@@ -772,8 +772,7 @@ struct prelim_drm_i915_debug_event {
#define PRELIM_DRM_I915_DEBUG_EVENT_CONTEXT_PARAM 7 #define PRELIM_DRM_I915_DEBUG_EVENT_CONTEXT_PARAM 7
#define PRELIM_DRM_I915_DEBUG_EVENT_EU_ATTENTION 8 #define PRELIM_DRM_I915_DEBUG_EVENT_EU_ATTENTION 8
#define PRELIM_DRM_I915_DEBUG_EVENT_ENGINES 9 #define PRELIM_DRM_I915_DEBUG_EVENT_ENGINES 9
#define PRELIM_DRM_I915_DEBUG_EVENT_PAGE_FAULT 10 #define PRELIM_DRM_I915_DEBUG_EVENT_MAX_EVENT PRELIM_DRM_I915_DEBUG_EVENT_ENGINES
#define PRELIM_DRM_I915_DEBUG_EVENT_MAX_EVENT PRELIM_DRM_I915_DEBUG_EVENT_PAGE_FAULT
__u32 flags; __u32 flags;
#define PRELIM_DRM_I915_DEBUG_EVENT_CREATE (1 << 31) #define PRELIM_DRM_I915_DEBUG_EVENT_CREATE (1 << 31)
@@ -871,34 +870,6 @@ struct prelim_drm_i915_debug_event_eu_attention {
__u8 bitmask[0]; __u8 bitmask[0];
} __attribute__((packed)); } __attribute__((packed));
struct prelim_drm_i915_debug_event_page_fault {
struct prelim_drm_i915_debug_event base;
__u64 client_handle;
__u64 ctx_handle;
__u64 lrc_handle;
__u32 flags;
struct i915_engine_class_instance ci;
__u64 page_fault_address;
/**
* Size of one bitmask: sum of size before/after/resolved att bits.
* It has three times the size of prelim_drm_i915_debug_event_eu_attention.bitmask_size.
*/
__u32 bitmask_size;
/**
* Bitmask of thread attentions starting from natural
* hardware order of slice=0,subslice=0,eu=0, 8 attention
* bits per eu.
* The order of the bitmask array is before, after, resolved.
*/
__u8 bitmask[0];
} __attribute__((packed));
struct prelim_drm_i915_debug_read_uuid { struct prelim_drm_i915_debug_read_uuid {
__u64 client_handle; __u64 client_handle;
__u64 handle; __u64 handle;