Media
/
gmmlib
mirror of https://github.com/intel/gmmlib.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add Pagetable manager and auxtable support. 

Change-Id: I429b52a70c77fc304c65d28d42b18f99b01116d5
This commit is contained in:
johnbasha shaik 2019-09-12 12:19:30 +05:30
parent b136069616
commit a09c68fd32
26 changed files with 4403 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
Source/GmmLib/CMakeLists.txt
View File

@ -191,6 +191,7 @@ set(HEADERS_
${BS_DIR_GMMLIB}/inc/External/Common/GmmConst.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmDebug.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmFormatTable.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmHw.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInfo.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInfoExt.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInternal.h
@ -205,6 +206,7 @@ set(HEADERS_
${BS_DIR_GMMLIB}/inc/External/Common/GmmTextureExt.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmUtil.h
${BS_DIR_GMMLIB}/inc/External/Linux/GmmResourceInfoLin.h
${BS_DIR_GMMLIB}/inc/External/Linux/GmmResourceInfoLinExt.h
${BS_DIR_GMMLIB}/inc/Internal/Common/Platform/GmmGen10Platform.h
${BS_DIR_GMMLIB}/inc/Internal/Common/Platform/GmmGen11Platform.h
${BS_DIR_GMMLIB}/inc/Internal/Common/Platform/GmmGen12Platform.h
@ -225,6 +227,7 @@ set(HEADERS_
set(UMD_HEADERS
${HEADERS_}
${BS_DIR_GMMLIB}/inc/External/Common/GmmPageTableMgr.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmClientContext.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmLibDll.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmLibDllName.h
@ -250,6 +253,7 @@ set(SOURCES_
${BS_DIR_GMMLIB}/Resource/GmmResourceInfoCommon.cpp
${BS_DIR_GMMLIB}/Resource/GmmResourceInfoCommonEx.cpp
${BS_DIR_GMMLIB}/Resource/GmmRestrictions.cpp
${BS_DIR_GMMLIB}/Resource/Linux/GmmResourceInfoLinCWrapper.cpp
${BS_DIR_GMMLIB}/Texture/GmmGen7Texture.cpp
${BS_DIR_GMMLIB}/Texture/GmmGen8Texture.cpp
${BS_DIR_GMMLIB}/Texture/GmmGen9Texture.cpp
@ -269,6 +273,9 @@ set(SOURCES_
set(UMD_SOURCES
${SOURCES_}
${BS_DIR_GMMLIB}/TranslationTable/GmmAuxTable.cpp
${BS_DIR_GMMLIB}/TranslationTable/GmmPageTableMgr.cpp
${BS_DIR_GMMLIB}/TranslationTable/GmmUmdTranslationTable.cpp
${BS_DIR_GMMLIB}/GlobalInfo/GmmClientContext.cpp
${BS_DIR_GMMLIB}/GlobalInfo/GmmOldApi.cpp
${BS_DIR_GMMLIB}/GlobalInfo/GmmLibDllMain.cpp
@ -287,13 +294,27 @@ source_group("Source Files\\Cache Policy" ${BS_DIR_GMMLIB}/CachePolicy/*.cpp)
source_group("Source Files\\Global" ${BS_DIR_GMMLIB}/GlobalInfo/.*)
source_group("Source Files\\Platform" ${BS_DIR_GMMLIB}/Platform/.*)
source_group("Source Files\\Texture" ${BS_DIR_GMMLIB}/Texture/.*)
source_group("Source Files\\Translation Table" ${BS_DIR_GMMLIB}/TranslationTable/.*)
source_group("Source Files\\Utility" ${BS_DIR_GMMLIB}/Utility/.*)
source_group("Source Files\\Resource" FILES
${BS_DIR_GMMLIB}/Resource/GmmResourceInfo.cpp
${BS_DIR_GMMLIB}/Resource/GmmResourceInfoCommon.cpp
${BS_DIR_GMMLIB}/Resource/GmmResourceInfoCommonEx.cpp
${BS_DIR_GMMLIB}/Resource/GmmRestrictions.cpp)
source_group("Source Files\\Resource\\Linux" FILES
${BS_DIR_GMMLIB}/Resource/Linux/GmmResourceInfoLinCWrapper.cpp
)
source_group("Source Files\\TranslationTable\\Windows" FILES
${BS_DIR_GMMLIB}/TranslationTable/GmmAuxTable.cpp
${BS_DIR_GMMLIB}/TranslationTable/GmmPageTableMgr.cpp
${BS_DIR_GMMLIB}/TranslationTable/GmmUmdTranslationTable.cpp)
source_group("Source Files\\TranslationTable" FILES
${BS_DIR_GMMLIB}/TranslationTable/GmmUmdTranslationTable.h)
source_group("Header Files\\External\\Common" FILES
${BS_DIR_GMMLIB}/inc/External/Common/GmmCachePolicy.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmCachePolicyCommon.h
@ -302,6 +323,7 @@ source_group("Header Files\\External\\Common" FILES
${BS_DIR_GMMLIB}/inc/External/Common/GmmConst.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmDebug.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmFormatTable.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmHw.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInfo.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInfoExt.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInternal.h
@ -331,6 +353,7 @@ source_group("Header Files\\External\\Common\\Cache Policy" FILES
source_group("Header Files\\External\\Linux" FILES
${BS_DIR_GMMLIB}/inc/External/Linux/GmmResourceInfoLin.h
${BS_DIR_GMMLIB}/inc/External/Linux/GmmResourceInfoLinExt.h
)
source_group("Header Files\\Internal\\Common" FILES

77
Source/GmmLib/GlobalInfo/GmmClientContext.cpp
View File

@ -27,6 +27,7 @@ OTHER DEALINGS IN THE SOFTWARE.
#include "..\..\inc\common\gfxEscape.h"
#include "..\..\..\miniport\LHDM\inc\gmmEscape.h"
#include "Internal\Windows\GmmResourceInfoWinInt.h"
#include "../TranslationTable/GmmUmdTranslationTable.h"
#endif
/////////////////////////////////////////////////////////////////////////////////////
@ -465,6 +466,40 @@ void GMM_STDCALL GmmLib::GmmClientContext::DestroyResInfoObject(GMM_RESOURCE_INF
}
}
/////////////////////////////////////////////////////////////////////////////////////
/// Member function of ClientContext class for creation of PAgeTableMgr Object .
/// @see GmmLib::GMM_PAGETABLE_MGR::GMM_PAGETABLE_MGR
///
/// @param[in] pDevCb: Pointer to GMM_DEVICE_CALLBACKS_INT
/// @param[in] pTTCB: Pointer to GMM_TRANSLATIONTABLE_CALLBACKS
/// @param[in] TTFags
/// @return Pointer to GMM_PAGETABLE_MGR class.
//TBD: move the code to new overloaded the API and remove this API once all clients are moved to new API.
/////////////////////////////////////////////////////////////////////////////////////
GMM_PAGETABLE_MGR* GMM_STDCALL GmmLib::GmmClientContext::CreatePageTblMgrObject(GMM_DEVICE_CALLBACKS_INT* pDevCb,
uint32_t TTFlags)
{
GMM_PAGETABLE_MGR* pPageTableMgr = NULL;
pPageTableMgr = new GMM_PAGETABLE_MGR(pDevCb, TTFlags, this);
return pPageTableMgr;
}
/////////////////////////////////////////////////////////////////////////////////////
/// Member function of ClientContext class for destroy of PageTableMgr Object .
///
/// @param[in] pPageTableMgr: Pointer to GMM_PAGETABLE_MGR
/// @return void.
/////////////////////////////////////////////////////////////////////////////////////
void GMM_STDCALL GmmLib::GmmClientContext::DestroyPageTblMgrObject(GMM_PAGETABLE_MGR* pPageTableMgr)
{
if (pPageTableMgr)
{
delete pPageTableMgr;
}
}
#ifdef GMM_LIB_DLL
/////////////////////////////////////////////////////////////////////////////////////
/// Member function of ClientContext class for creation of ResourceInfo Object .
@ -568,6 +603,48 @@ void GMM_STDCALL GmmLib::GmmClientContext::DestroyResInfoObject(GMM_RESOURCE_INF
}
}
#endif
/////////////////////////////////////////////////////////////////////////////////////
/// Member function of ClientContext class for creation of PAgeTableMgr Object .
/// @see GmmLib::GMM_PAGETABLE_MGR::GMM_PAGETABLE_MGR
///
/// @param[in] pDevCb: Pointer to GMM_DEVICE_CALLBACKS_INT
/// @param[in] pTTCB: Pointer to GMM_TRANSLATIONTABLE_CALLBACKS
/// @param[in] TTFags
/// @return Pointer to GMM_PAGETABLE_MGR class.
/// TBD: move the code to new overloaded the API and remove this API once all clients are moved to new API.
/////////////////////////////////////////////////////////////////////////////////////
GMM_PAGETABLE_MGR* GMM_STDCALL GmmLib::GmmClientContext::CreatePageTblMgrObject(
GMM_DEVICE_CALLBACKS_INT* pDevCb,
uint32_t TTFlags,
GmmClientAllocationCallbacks* pAllocCbs)
{
if (!pAllocCbs || !pAllocCbs->pfnAllocation)
{
return CreatePageTblMgrObject(
pDevCb,
TTFlags);
}
else
{
GMM_PAGETABLE_MGR* pPageTableMgr = NULL;
return pPageTableMgr;
}
}
/////////////////////////////////////////////////////////////////////////////////////
/// Member function of ClientContext class for destroy of PageTableMgr Object .
///
/// @param[in] pPageTableMgr: Pointer to GMM_PAGETABLE_MGR
/// @return void.
/////////////////////////////////////////////////////////////////////////////////////
void GMM_STDCALL GmmLib::GmmClientContext::DestroyPageTblMgrObject(GMM_PAGETABLE_MGR* pPageTableMgr,
GmmClientAllocationCallbacks* pAllocCbs)
{
if (!pAllocCbs || !pAllocCbs->pfnFree)
{
return DestroyPageTblMgrObject(pPageTableMgr);
}
}
/////////////////////////////////////////////////////////////////////////////////////
/// Gmm lib DLL exported C wrapper for creating GmmLib::GmmClientContext object

2
Source/GmmLib/Resource/GmmRestrictions.cpp
View File

@ -577,7 +577,7 @@ void GmmLib::GmmTextureCalc::GetResRestrictions(GMM_TEXTURE_INFO * pTexinfo,
if(pTexinfo->Flags.Info.RenderCompressed ||
pTexinfo->Flags.Info.MediaCompressed)
{
Restrictions.Alignment = GFX_ALIGN(Restrictions.Alignment, GMM_KBYTE(64));
Restrictions.Alignment = GFX_ALIGN(Restrictions.Alignment, (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)));
}
GMM_DPF_EXIT;

179
Source/GmmLib/Resource/Linux/GmmResourceInfoLinCWrapper.cpp Normal file
View File

@ -0,0 +1,179 @@
/*==============================================================================
Copyright(c) 2019 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files(the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and / or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
============================================================================*/
#include "Internal/Common/GmmLibInc.h"
#if defined(__linux__) && !LHDM
#include "Internal/Linux/GmmResourceInfoLinInt.h"
/////////////////////////////////////////////////////////////////////////////////////
/// This is an overloaded function to call DeviceCb Function for CreateAllocation
///
/// @param[in] ClientType
/// @param[in] pDeviceCb: Pointer to GMM_DEVICE_CALLBACKS_INT Struct
/// @param[in] pAllocate: Pointer to GMM_DDI_ALLOCATE Union
/// @return Status of CreateAllocation call.
/////////////////////////////////////////////////////////////////////////////////////
int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_ALLOCATE *pAllocate)
{
int Status = 0;
void * pBo = NULL;
void * pCpuAddr = NULL;
uint64_t gpuAddr = 0ULL;
Status = pDeviceCb->DevCbPtrs_.pfnAllocate(pDeviceCb->pBufMgr,
pAllocate->size,
pAllocate->alignment,
&pBo,
&pCpuAddr,
&gpuAddr);
pAllocate->bo = pBo;
pAllocate->cpuAddr = pCpuAddr;
pAllocate->gfxAddr = gpuAddr;
return Status;
}
/////////////////////////////////////////////////////////////////////////////////////
/// This is an overloaded function to call DeviceCb Function for DestroyAllocation
///
/// @param[in] ClientType
/// @param[in] pDeviceCb: Pointer to GMM_DEVICE_CALLBACKS_INT Struct
/// @param[in] pAllocate: Pointer to GMM_DDI_DEALLOCATE Union
/// @return Status of DestroyAllocation call.
/////////////////////////////////////////////////////////////////////////////////////
int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_DEALLOCATE *pDeallocate)
{
int Status = 0;
pDeviceCb->DevCbPtrs_.pfnDeallocate(pDeallocate->bo);
return Status;
}
/////////////////////////////////////////////////////////////////////////////////////
/// This is an overloaded function to call DeviceCb Function for WaitForSyncObjFromCpu
///
/// @param[in] ClientType
/// @param[in] pDeviceCb: Pointer to GMM_DEVICE_CALLBACKS_INT Struct
/// @param[in] pAllocate: Pointer to GMM_DDI_WAITFORSYNCHRONIZATIONOBJECTFROMCPU Union
/// @return Status of WaitForSyncObjFromCpu call.
/////////////////////////////////////////////////////////////////////////////////////
int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_WAITFORSYNCHRONIZATIONOBJECTFROMCPU *pWait)
{
int Status = 0;
pDeviceCb->DevCbPtrs_.pfnWaitFromCpu(pWait->bo);
return Status;
}
/////////////////////////////////////////////////////////////////////////////////////
/// This function checks for Null DeviceCb Function pointer
///
/// @param[in] ClientType
/// @param[in] pDeviceCb: Pointer to GMM_DEVICE_CALLBACKS_INT Struct
/// @param[in] CallBackType Enum @GMM_DEVICE_CALLBACKS_TYPE
/// @return True if not Null.
/////////////////////////////////////////////////////////////////////////////////////
int GmmCheckForNullDevCbPfn(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DEVICE_CALLBACKS_TYPE CallBackType)
{
int Status = 0;
switch(CallBackType)
{
case GMM_DEV_CB_ALLOC:
Status = (pDeviceCb->DevCbPtrs_.pfnAllocate != 0);
break;
case GMM_DEV_CB_DEALLOC:
Status = (pDeviceCb->DevCbPtrs_.pfnDeallocate != 0);
break;
case GMM_DEV_CB_WAIT_FROM_CPU:
Status = (pDeviceCb->DevCbPtrs_.pfnWaitFromCpu != 0);
break;
default:
Status = 0;
break;
}
return Status;
}
// Dummy Translation Table Callback for reusing ..
static inline int DummyPrologTranslationTable(void *pDeviceHandle)
{
return 0;
}
static inline int DummyWriteL1Entries(void * pDeviceHandle,
const uint32_t NumEntries,
GMM_GFX_ADDRESS GfxAddress,
uint32_t * Data)
{
return 0;
}
static inline int DummyWriteL2L3Entry(void * pDeviceHandle,
GMM_GFX_ADDRESS GfxAddress,
uint64_t Data)
{
return 0;
}
static inline int DummyWriteFenceID(void * pDeviceHandle,
GMM_GFX_ADDRESS GfxAddress,
uint64_t Data)
{
return 0;
}
static inline int DummyEpilogTranslationTable(void * pDeviceHandle,
uint8_t ForceFlush)
{
return 0;
}
static inline int DummyCopyL1Entry(void * pDeviceHandle,
GMM_GFX_ADDRESS DstGfxAddress,
GMM_GFX_ADDRESS SrcGfxAddress)
{
return 0;
}
static inline int DummyWriteL3Adr(void * pDeviceHandle,
GMM_GFX_ADDRESS L3GfxAddress,
uint64_t RegOffset)
{
return 0;
}
GMM_TRANSLATIONTABLE_CALLBACKS DummyTTCB = {
.pfPrologTranslationTable = DummyPrologTranslationTable,
.pfWriteL1Entries = DummyWriteL1Entries,
.pfWriteL2L3Entry = DummyWriteL2L3Entry,
.pfWriteFenceID = DummyWriteFenceID,
.pfEpilogTranslationTable = DummyEpilogTranslationTable,
.pfCopyL1Entry = DummyCopyL1Entry,
.pfWriteL3Adr = DummyWriteL3Adr,
};
#endif /*__linux__*/

9
Source/GmmLib/Texture/GmmGen11Texture.cpp
View File

@ -321,9 +321,12 @@ void GmmLib::GmmGen11TextureCalc::FillPlanarOffsetAddress(GMM_TEXTURE_INFO *pTex
GMM_GFX_SIZE_T TileHeight = pPlatform->TileInfo[pTexInfo->TileMode].LogicalTileHeight;
GMM_GFX_SIZE_T TileWidth = pPlatform->TileInfo[pTexInfo->TileMode].LogicalTileWidth;
//U/V must be aligned to AuxT granularity, for 16K AuxT- 4x pitchalign enforces it,
//add extra padding for 64K AuxT
TileHeight *= 1;
if(GFX_GET_CURRENT_RENDERCORE(pPlatform->Platform) > IGFX_GEN11LP_CORE)
{
//U/V must be aligned to AuxT granularity, for 16K AuxT- 4x pitchalign enforces it,
//add extra padding for 64K AuxT
TileHeight *= (!GMM_IS_64KB_TILE(pTexInfo->Flags) && !WA16K) ? 4 : 1;
}
*pUOffsetX = GFX_ALIGN(*pUOffsetX, TileWidth);
*pUOffsetY = GFX_ALIGN(*pUOffsetY, TileHeight);

2
Source/GmmLib/Texture/GmmGen12Texture.cpp
View File

@ -838,7 +838,7 @@ GMM_STATUS GMM_STDCALL GmmLib::GmmGen12TextureCalc::FillTexPlanar(GMM_TEXTURE_IN
//U/V must be aligned to AuxT granularity, 4x pitchalign enforces 16K-align,
//add extra padding for 64K AuxT
TileHeight *= (!GMM_IS_64KB_TILE(pTexInfo->Flags)) ? 4 : 1;
TileHeight *= (!GMM_IS_64KB_TILE(pTexInfo->Flags) && !WA16K) ? 4 : 1;
if(pTexInfo->Format == GMM_FORMAT_IMC2 || // IMC2, IMC4 needs even tile columns
pTexInfo->Format == GMM_FORMAT_IMC4)

949
Source/GmmLib/TranslationTable/GmmAuxTable.cpp Normal file
View File

@ -0,0 +1,949 @@
/*==============================================================================
Copyright(c) 2019 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files(the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and / or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
Description: AUX-Table management functions
(contains functions to assign memory to
AUX-Tables with valid entries,
and update their entries on request)
============================================================================*/
#include "Internal/Common/GmmLibInc.h"
#include "../TranslationTable/GmmUmdTranslationTable.h"
#if !defined(__GMM_KMD__)
//=============================================================================
//
// Function: MapNullCCS
//
// Desc: Maps given resource, with dummy null-ccs chain, on Aux Table
//
// Caller: UpdateAuxTable (map op for null-tiles)
//
// Parameters:
// UmdContext: Caller-thread specific info (regarding BB for TR-Aux udpate, cmdQ to use etc)
// BaseAdr: Start adr of main surface
// Size: Main-surface size in bytes
// PartialL1e: Aux-metadata other than AuxVA
// DoNotWait: 1 for CPU update, 0 for async(Gpu) update
//-----------------------------------------------------------------------------
GMM_STATUS GmmLib::AuxTable::MapNullCCS(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint64_t PartialL1e, uint8_t DoNotWait)
{
GMM_STATUS Status = GMM_SUCCESS;
GMM_GFX_SIZE_T L1TableSize = (GMM_L1_SIZE(AUXTT, pGmmGlobalContext)) * (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)); //Each AuxTable entry maps 16K main-surface
GMM_GFX_ADDRESS Addr = 0;
GMM_GFX_ADDRESS L3GfxAddress = 0;
GMM_CLIENT ClientType;
GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
EnterCriticalSection(&TTLock);
DoNotWait |= (!UmdContext || !UmdContext->pCommandQueueHandle);
if(TTL3.L3Handle)
{
L3GfxAddress = TTL3.GfxAddress;
}
else
{
LeaveCriticalSection(&TTLock);
return GMM_ERROR;
}
if(!DoNotWait)
{
PageTableMgr->TTCb.pfPrologTranslationTable(
UmdContext->pCommandQueueHandle);
}
// For each L1 table
for(Addr = GFX_ALIGN_FLOOR(BaseAdr, L1TableSize); // Start at begining of L1 table
Addr < BaseAdr + Size;
Addr += L1TableSize) // Increment by 1 L1 table
{
GMM_GFX_ADDRESS L1GfxAddress, L2GfxAddress;
GMM_GFX_ADDRESS L1CPUAddress, L2CPUAddress;
GMM_GFX_ADDRESS StartAddress = 0;
GMM_GFX_ADDRESS EndAddress = 0;
GMM_GFX_ADDRESS TileAddr = 0;
GMM_GFX_SIZE_T L2eIdx = 0;
StartAddress = Addr < BaseAdr ? BaseAdr : Addr;
EndAddress = Addr + L1TableSize;
if(EndAddress > BaseAdr + Size)
{
EndAddress = BaseAdr + Size;
}
GetL1L2TableAddr(StartAddress,
&L1GfxAddress,
&L2GfxAddress);
// If tables are not there, then they are already invalidated as part of
// AUX-TT initialization or other APIs.
if(L2GfxAddress == GMM_NO_TABLE ||
L1GfxAddress == GMM_NO_TABLE)
{
//Clear Valid-bit for L3Entry or L2Entry
uint64_t Data = 0;
GMM_GFX_ADDRESS TableGfxAddress = (L2GfxAddress == GMM_NO_TABLE) ? L3GfxAddress : L2GfxAddress;
GMM_GFX_ADDRESS TableCPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? TTL3.CPUAddress : pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].GetCPUAddress();
uint32_t TableEntryIdx = (L2GfxAddress == GMM_NO_TABLE) ? static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress)) : static_cast<uint32_t>(GMM_L2_ENTRY_IDX(AUXTT, StartAddress));
L2CPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? 0 : TableCPUAddress;
if(!NullL1Table || !NullL2Table)
{
AllocateDummyTables(&NullL2Table, &NullL1Table);
if(!NullL1Table || !NullL2Table)
{
//report error
LeaveCriticalSection(&TTLock);
return GMM_OUT_OF_MEMORY;
}
else
{
//Initialize dummy table entries (one-time)
GMM_GFX_ADDRESS TableAddr = NullL2Table->GetCPUAddress();
GMM_AUXTTL2e L2e = {0};
L2e.Valid = 1;
L2e.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
for(int i = 0; i < GMM_AUX_L2_SIZE; i++)
{
//initialize L2e ie clear Valid bit for all entries
((GMM_AUXTTL2e *)TableAddr)[i].Value = L2e.Value;
}
TableAddr = NullL1Table->GetCPUAddress();
GMM_AUXTTL1e L1e = {0};
L1e.Valid = 1;
L1e.GfxAddress = (NullCCSTile >> 8);
for(int i = 0; i < GMM_AUX_L1_SIZE(pGmmGlobalContext); i++)
{
//initialize L1e with null ccs tile
((GMM_AUXTTL1e *)TableAddr)[i].Value = L1e.Value;
}
}
}
if(L2GfxAddress == GMM_NO_TABLE)
{
GMM_AUXTTL3e L3e = {0};
L3e.Valid = 1;
L3e.L2GfxAddr = (NullL2Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL2Table->GetNodeIdx()) >> 15;
Data = L3e.Value;
}
else
{
GMM_AUXTTL2e L2e = {0};
L2e.Valid = 1;
L2e.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
Data = L2e.Value;
}
if(DoNotWait)
{
//Sync update on CPU
((GMM_AUXTTL2e *)TableCPUAddress)[TableEntryIdx].Value = Data;
}
else
{
if(L2GfxAddress != GMM_NO_TABLE)
{
pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].UpdatePoolFence(UmdContext, false);
}
PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
TableGfxAddress + TableEntryIdx * GMM_AUX_L2e_SIZE,
Data);
}
continue;
}
else
{
uint32_t L3eIdx = static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress));
L2CPUAddress = pTTL2[L3eIdx].GetCPUAddress();
L2eIdx = GMM_L2_ENTRY_IDX(AUXTT, StartAddress);
if(DoNotWait)
{
//Sync update on CPU
((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Valid = 1; //set Valid bit
((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].L2GfxAddr = L2GfxAddress >> 15;
((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Valid = 1; //set Valid bit
((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].L1GfxAddr = L1GfxAddress >> 13;
}
else
{
GMM_AUXTTL3e L3e = {0};
L3e.Valid = 1;
L3e.L2GfxAddr = L2GfxAddress >> 15;
PageTableMgr->TTCb.pfWriteL2L3Entry(
UmdContext->pCommandQueueHandle,
L3GfxAddress + (L3eIdx * GMM_AUX_L3e_SIZE),
L3e.Value);
pTTL2[L3eIdx].UpdatePoolFence(UmdContext, false);
GMM_AUXTTL2e L2e = {0};
L2e.Valid = 1;
L2e.L1GfxAddr = L1GfxAddress >> 13;
PageTableMgr->TTCb.pfWriteL2L3Entry(
UmdContext->pCommandQueueHandle,
L2GfxAddress + (L2eIdx * GMM_AUX_L2e_SIZE),
L2e.Value);
}
}
// For each 64KB or 16KB of main surface (entry) in L1 table
for(TileAddr = StartAddress;
TileAddr < EndAddress;
TileAddr += (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)))
{
uint64_t Data = PartialL1e | NullCCSTile | __BIT(0);
GMM_GFX_SIZE_T L1eIdx = GMM_L1_ENTRY_IDX(AUXTT, TileAddr, pGmmGlobalContext);
GmmLib::LastLevelTable *pL1Tbl = NULL;
pL1Tbl = pTTL2[GMM_AUX_L3_ENTRY_IDX(TileAddr)].GetL1Table(L2eIdx, NULL);
L1CPUAddress = pL1Tbl->GetCPUAddress();
if(DoNotWait)
{
//Sync update on CPU
((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx].Value = Data;
GMM_DPF(GFXDBG_CRITICAL, "Null-Map | Table Entry: [0x%06x] L2Addr[0x%016llX] Value[0x%016llX] :: [0x%06x] L1Addr[0x%016llX] Value[0x%016llX]\n", L2eIdx, ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx], ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].L1GfxAddr << 13, L1eIdx, &((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx], Data);
}
else
{
pL1Tbl->UpdatePoolFence(UmdContext, false);
/* PageTableMgr->TTCb.pfWriteL1Entries(
UmdContext->pCommandQueueHandle,
2,
L1GfxAddress + (L1eIdx * GMM_AUX_L1e_SIZE),
(uint32_t*)(&Data));*/ //**********REQUIRE UMD CHANGE TO UPDATE 64-bit ENTRY - both DWORDs must be updated atomically*******/
PageTableMgr->TTCb.pfWriteL2L3Entry(
UmdContext->pCommandQueueHandle,
L1GfxAddress + (L1eIdx * GMM_AUX_L1e_SIZE),
Data);
}
if(pL1Tbl->TrackTableUsage(AUXTT, true, TileAddr, true))
{ // L1 Table is not being used anymore
GMM_AUXTTL2e L2e = {0};
GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
GmmLib::LastLevelTable * pL1Tbl = NULL, *Prev = NULL;
pL1Tbl = pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].GetL1Table(L2eIdx, &Prev);
// Map L2-entry to Null-L1Table
L2e.Valid = 1;
L2e.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
if(DoNotWait)
{
//Sync update on CPU
((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Value = L2e.Value;
}
else
{
pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].UpdatePoolFence(UmdContext, false);
PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
L2GfxAddress + L2eIdx * GMM_AUX_L2e_SIZE,
L2e.Value);
}
//Update usage for PoolNode assigned to L1Table, and free L1Tbl
if(pL1Tbl)
{
PoolElem = pL1Tbl->GetPool();
if(PoolElem)
{
if(pL1Tbl->GetBBInfo().BBQueueHandle)
{
PoolElem->GetNodeBBInfoAtIndex(pL1Tbl->GetNodeIdx()) = pL1Tbl->GetBBInfo();
}
DEASSIGN_POOLNODE(PageTableMgr, UmdContext, PoolElem, pL1Tbl->GetNodeIdx(), AUX_L1TABLE_SIZE_IN_POOLNODES)
}
pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].DeleteFromList(pL1Tbl, Prev);
}
// The L1 table is unused -- meaning everything else in this table is
// already invalid. So, break early.
break;
}
}
}
if(!DoNotWait)
{
PageTableMgr->TTCb.pfEpilogTranslationTable(
UmdContext->pCommandQueueHandle,
1); // ForceFlush
}
LeaveCriticalSection(&TTLock);
return Status;
}
//=============================================================================
//
// Function: InvalidateTable (InvalidateMappings)
//
// Desc: Unmaps given resource from Aux Table; and marks affected entries as invalid
//
// Caller: UpdateAuxTable (unmap op)
//
// Parameters:
// UmdContext: Caller-thread specific info (regarding BB for Aux udpate, cmdQ to use etc)
// BaseAdr: Start adr of main surface
// Size: Main-surface size in bytes? (or take GmmResInfo?)
// DoNotWait: 1 for CPU update, 0 for async(Gpu) update
//-----------------------------------------------------------------------------
GMM_STATUS GmmLib::AuxTable::InvalidateTable(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint8_t DoNotWait)
{
GMM_STATUS Status = GMM_SUCCESS;
GMM_GFX_SIZE_T L1TableSize = (GMM_L1_SIZE(AUXTT, pGmmGlobalContext)) * (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)); //Each AuxTable entry maps 16K main-surface
GMM_GFX_ADDRESS Addr = 0;
GMM_GFX_ADDRESS L3GfxAddress = 0;
uint8_t isTRVA = 0;
GMM_CLIENT ClientType;
GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
//NullCCSTile isn't initialized, disable TRVA path
isTRVA = (NullCCSTile ? isTRVA : 0);
EnterCriticalSection(&TTLock);
DoNotWait |= (!UmdContext || !UmdContext->pCommandQueueHandle);
if(TTL3.L3Handle)
{
L3GfxAddress = TTL3.GfxAddress;
}
else
{
LeaveCriticalSection(&TTLock);
return GMM_ERROR;
}
if(!DoNotWait)
{
PageTableMgr->TTCb.pfPrologTranslationTable(
UmdContext->pCommandQueueHandle);
}
// For each L1 table
for(Addr = GFX_ALIGN_FLOOR(BaseAdr, L1TableSize); // Start at begining of L1 table
Addr < BaseAdr + Size;
Addr += L1TableSize) // Increment by 1 L1 table
{
GMM_GFX_ADDRESS L1GfxAddress, L2GfxAddress;
GMM_GFX_ADDRESS L1CPUAddress, L2CPUAddress;
GMM_GFX_ADDRESS StartAddress = 0;
GMM_GFX_ADDRESS EndAddress = 0;
GMM_GFX_ADDRESS TileAddr = 0;
GMM_GFX_SIZE_T L2eIdx = 0;
StartAddress = Addr < BaseAdr ? BaseAdr : Addr;
EndAddress = Addr + L1TableSize;
if(EndAddress > BaseAdr + Size)
{
EndAddress = BaseAdr + Size;
}
GetL1L2TableAddr(StartAddress,
&L1GfxAddress,
&L2GfxAddress);
// If tables are not there, then they are already invalidated as part of
// AUX-TT initialization or other APIs.
if(L2GfxAddress == GMM_NO_TABLE ||
L1GfxAddress == GMM_NO_TABLE)
{
//Clear Valid-bit for L3Entry or L2Entry
GMM_AUXTTL2e L2e = {0}; //AUXTT L3e is identical to L2e, reuse.
GMM_GFX_ADDRESS TableGfxAddress = (L2GfxAddress == GMM_NO_TABLE) ? L3GfxAddress : L2GfxAddress;
GMM_GFX_ADDRESS TableCPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? TTL3.CPUAddress : pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].GetCPUAddress();
uint32_t TableEntryIdx = (L2GfxAddress == GMM_NO_TABLE) ? static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress)) : static_cast<uint32_t>(GMM_L2_ENTRY_IDX(AUXTT, StartAddress));
L2CPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? 0 : TableCPUAddress;
if(isTRVA && NullL2Table && NullL1Table)
{
//invalidate if request spans entire stretch ie TileAdr aligns L1TableSize*GMM_L2_SIZE
uint64_t Data = 0;
if(L2GfxAddress == GMM_NO_TABLE)
{
GMM_AUXTTL3e L3e = {0};
L3e.Valid = 1;
L3e.L2GfxAddr = (NullL2Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL2Table->GetNodeIdx()) >> 15;
Data = L3e.Value;
}
else
{
GMM_AUXTTL2e L2e = {0};
L2e.Valid = 1;
L2e.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
Data = L2e.Value;
}
L2e.Value = Data;
}
else
{
L2e.Valid = 0;
}
if(DoNotWait)
{
//Sync update on CPU
((GMM_AUXTTL2e *)TableCPUAddress)[TableEntryIdx].Value = L2e.Value;
}
else
{
if(L2GfxAddress != GMM_NO_TABLE)
{
pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].UpdatePoolFence(UmdContext, false);
}
PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
TableGfxAddress + TableEntryIdx * GMM_AUX_L2e_SIZE,
L2e.Value);
}
continue;
}
else
{
uint32_t L3eIdx = static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress));
L2CPUAddress = pTTL2[L3eIdx].GetCPUAddress();
L2eIdx = GMM_L2_ENTRY_IDX(AUXTT, StartAddress);
if(DoNotWait)
{
//Sync update on CPU
((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Valid = 1; //set Valid bit
((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].L2GfxAddr = L2GfxAddress >> 15;
((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Valid = 1; //set Valid bit
((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].L1GfxAddr = L1GfxAddress >> 13;
}
else
{
GMM_AUXTTL3e L3e = {0};
L3e.Valid = 1;
L3e.L2GfxAddr = L2GfxAddress >> 15;
PageTableMgr->TTCb.pfWriteL2L3Entry(
UmdContext->pCommandQueueHandle,
L3GfxAddress + (L3eIdx * GMM_AUX_L3e_SIZE),
L3e.Value);
pTTL2[L3eIdx].UpdatePoolFence(UmdContext, false);
GMM_AUXTTL2e L2e = {0};
L2e.Valid = 1;
L2e.L1GfxAddr = L1GfxAddress >> 13;
PageTableMgr->TTCb.pfWriteL2L3Entry(
UmdContext->pCommandQueueHandle,
L2GfxAddress + (L2eIdx * GMM_AUX_L2e_SIZE),
L2e.Value);
}
}
// For each 64KB or 16KB of main surface (entry) in L1 table
for(TileAddr = StartAddress;
TileAddr < EndAddress;
TileAddr += (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)))
{
//Invalidation of requested range irrespective of TRVA
uint64_t Data = GMM_INVALID_AUX_ENTRY;
GMM_GFX_SIZE_T L1eIdx = GMM_L1_ENTRY_IDX(AUXTT, TileAddr, pGmmGlobalContext);
GmmLib::LastLevelTable *pL1Tbl = NULL;
pL1Tbl = pTTL2[GMM_AUX_L3_ENTRY_IDX(TileAddr)].GetL1Table(L2eIdx, NULL);
L1CPUAddress = pL1Tbl->GetCPUAddress();
if(DoNotWait)
{
//Sync update on CPU
((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx].Value = Data;
GMM_DPF(GFXDBG_CRITICAL, "UnMap | Table Entry: [0x%06x] L2Addr[0x%016llX] Value[0x%016llX] :: [0x%06x] L1Addr[0x%016llX] Value[0x%016llX]\n", L2eIdx, ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx], ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].L1GfxAddr << 13, L1eIdx, &((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx], Data);
}
else
{
pL1Tbl->UpdatePoolFence(UmdContext, false);
/* PageTableMgr->TTCb.pfWriteL1Entries(
UmdContext->pCommandQueueHandle,
2,
L1GfxAddress + (L1eIdx * GMM_AUX_L1e_SIZE),
(uint32_t*)(&Data));*/ //**********REQUIRE UMD CHANGE TO UPDATE 64-bit ENTRY - both DWORDs must be updated atomically*******/
PageTableMgr->TTCb.pfWriteL2L3Entry(
UmdContext->pCommandQueueHandle,
L1GfxAddress + (L1eIdx * GMM_AUX_L1e_SIZE),
Data);
}
if(pL1Tbl->TrackTableUsage(AUXTT, true, TileAddr, true))
{ // L1 Table is not being used anymore
GMM_AUXTTL2e L2e = {0};
GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
GmmLib::LastLevelTable * pL1Tbl = NULL, *Prev = NULL;
pL1Tbl = pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].GetL1Table(L2eIdx, &Prev);
if(isTRVA && NullL1Table &&
((TileAddr > GFX_ALIGN_FLOOR(BaseAdr, L1TableSize) && TileAddr < GFX_ALIGN_NP2(BaseAdr, L1TableSize)) ||
(TileAddr > GFX_ALIGN_FLOOR(BaseAdr + Size, L1TableSize) && TileAddr < GFX_ALIGN_NP2(BaseAdr + Size, L1TableSize))))
{
//Invalidation affects entries out of requested range, null-map for TR
L2e.Valid = 1;
L2e.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
}
else
{
// Clear valid bit of L2 entry
L2e.Valid = 0;
((GMM_AUXTTL2e *)L2GfxAddress)[L2eIdx].Valid = 0;
}
if(DoNotWait)
{
//Sync update on CPU
((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Value = L2e.Value;
}
else
{
pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].UpdatePoolFence(UmdContext, false);
PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
L2GfxAddress + L2eIdx * GMM_AUX_L2e_SIZE,
L2e.Value);
}
//Update usage for PoolNode assigned to L1Table, and free L1Tbl
if(pL1Tbl)
{
PoolElem = pL1Tbl->GetPool();
if(PoolElem)
{
if(pL1Tbl->GetBBInfo().BBQueueHandle)
{
PoolElem->GetNodeBBInfoAtIndex(pL1Tbl->GetNodeIdx()) = pL1Tbl->GetBBInfo();
}
DEASSIGN_POOLNODE(PageTableMgr, UmdContext, PoolElem, pL1Tbl->GetNodeIdx(), AUX_L1TABLE_SIZE_IN_POOLNODES)
}
pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].DeleteFromList(pL1Tbl, Prev);
}
// The L1 table is unused -- meaning everything else in this table is
// already invalid. So, break early.
break;
}
}
}
if(!DoNotWait)
{
PageTableMgr->TTCb.pfEpilogTranslationTable(
UmdContext->pCommandQueueHandle,
1); // ForceFlush
}
LeaveCriticalSection(&TTLock);
return Status;
}
//=============================================================================
//
// Function: MapValidEntry
//
// Desc: Maps given main-surface, on Aux-Table, to get the exact CCS cacheline tied to
// different 4x4K pages of main-surface
//
// Caller: UpdateAuxTable (map op)
//
// Parameters:
// UmdContext: ptr to thread-data
// BaseAdr: Start adr of main-surface
// BaseSize: main-surface Size in bytes
// BaseResInfo: main surface ResInfo
// AuxVA: Start adr of Aux-surface
// AuxResInfo: Aux surface ResInfo
// PartialData: Aux L1 partial data (ie w/o address)
// DoNotWait: true for CPU update, false for async(Gpu) update
//-----------------------------------------------------------------------------
GMM_STATUS GmmLib::AuxTable::MapValidEntry(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T BaseSize,
GMM_RESOURCE_INFO *BaseResInfo, GMM_GFX_ADDRESS AuxVA, GMM_RESOURCE_INFO *AuxResInfo, uint64_t PartialData, uint8_t DoNotWait)
{
GMM_STATUS Status = GMM_SUCCESS;
GMM_GFX_ADDRESS Addr = 0, L3TableAdr = GMM_NO_TABLE;
GMM_GFX_SIZE_T L1TableSize = GMM_AUX_L1_SIZE(pGmmGlobalContext) * (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16));
GMM_GFX_SIZE_T CCS$Adr = AuxVA;
uint8_t isTRVA =0 ;
GMM_CLIENT ClientType;
GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
//NullCCSTile isn't initialized, disable TRVA path
isTRVA = (NullCCSTile ? isTRVA : 0);
EnterCriticalSection(&TTLock);
if(!TTL3.L3Handle || (!DoNotWait && !UmdContext))
{
Status = GMM_ERROR;
}
else
{
L3TableAdr = TTL3.GfxAddress;
if(!DoNotWait)
{
PageTableMgr->TTCb.pfPrologTranslationTable(UmdContext->pCommandQueueHandle);
}
GMM_DPF(GFXDBG_CRITICAL, "Mapping surface: GPUVA=0x%016llX Size=0x%08X Aux_GPUVA=0x%016llX\n", BaseAdr, BaseSize, AuxVA);
for(Addr = GFX_ALIGN_FLOOR(BaseAdr, L1TableSize); Addr < BaseAdr + BaseSize; Addr += L1TableSize)
{
GMM_GFX_ADDRESS StartAdr, EndAdr, TileAdr;
GMM_GFX_ADDRESS L1TableAdr = GMM_NO_TABLE, L2TableAdr = GMM_NO_TABLE;
GMM_GFX_ADDRESS L1TableCPUAdr = GMM_NO_TABLE, L2TableCPUAdr = GMM_NO_TABLE;
GMM_GFX_SIZE_T L2eIdx = 0;
GMM_GFX_SIZE_T L3eIdx = 0;
bool AllocateL1 = false, AllocateL2 = false;
EndAdr = Addr + L1TableSize;
EndAdr = EndAdr > BaseAdr + BaseSize ? BaseAdr + BaseSize : EndAdr;
StartAdr = Addr < BaseAdr ? BaseAdr : Addr;
L2eIdx = GMM_L2_ENTRY_IDX(AUXTT, StartAdr);
L3eIdx = GMM_L3_ENTRY_IDX(AUXTT, StartAdr);
//Allocate L2/L1 Table -- get L2 Table Adr for <StartAdr,EndAdr>
GetL1L2TableAddr(Addr, &L1TableAdr, &L2TableAdr);
if(L2TableAdr == GMM_NO_TABLE || L1TableAdr == GMM_NO_TABLE)
{
AllocateL1 = GMM_NO_TABLE == L1TableAdr;
AllocateL2 = GMM_NO_TABLE == L2TableAdr;
AllocateL1L2Table(Addr, &L1TableAdr, &L2TableAdr);
if(L2TableAdr == GMM_NO_TABLE || L1TableAdr == GMM_NO_TABLE)
{
LeaveCriticalSection(&TTLock);
return GMM_OUT_OF_MEMORY;
}
if(AllocateL2)
{
uint32_t i = 0;
GMM_AUXTTL2e InvalidEntry;
InvalidEntry.Value = 0;
if(isTRVA && NullL1Table)
{
InvalidEntry.Valid = 1;
InvalidEntry.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
}
if(DoNotWait)
{
L2TableCPUAdr = pTTL2[L3eIdx].GetCPUAddress();
((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Value = 0;
((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].L2GfxAddr = L2TableAdr >> 15;
((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Valid = 1;
for(i = 0; i < GMM_AUX_L2_SIZE; i++)
{
//initialize L2e ie clear Valid bit for all entries
((GMM_AUXTTL2e *)L2TableCPUAdr)[i].Value = InvalidEntry.Value;
}
}
else
{
GMM_AUXTTL3e L3e = {0};
L3e.Valid = 1;
L3e.L2GfxAddr = L2TableAdr >> 15;
PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
L3TableAdr + L3eIdx * GMM_AUX_L3e_SIZE,
L3e.Value);
//initialize L2e ie clear valid bit for all entries
for(i = 0; i < GMM_AUX_L2_SIZE; i++)
{
PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
L2TableAdr + i * GMM_AUX_L2e_SIZE,
InvalidEntry.Value);
}
}
}
if(AllocateL1)
{
uint64_t InvalidEntry = (!isTRVA) ? GMM_INVALID_AUX_ENTRY : (NullCCSTile | __BIT(0));
uint32_t i = 0;
if(DoNotWait)
{
GmmLib::LastLevelTable *pL1Tbl = NULL;
pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx, NULL);
L2TableCPUAdr = pTTL2[L3eIdx].GetCPUAddress();
L1TableCPUAdr = pL1Tbl->GetCPUAddress();
//Sync update on CPU
((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx].Value = 0;
((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx].L1GfxAddr = L1TableAdr >> 13;
((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx].Valid = 1;
for(i = 0; i < (uint32_t)GMM_AUX_L1_SIZE(pGmmGlobalContext); i++)
{
//initialize L1e ie mark all entries with Null tile value
((GMM_AUXTTL1e *)L1TableCPUAdr)[i].Value = InvalidEntry;
}
}
else
{
GMM_AUXTTL2e L2e = {0};
L2e.Valid = 1;
L2e.L1GfxAddr = L1TableAdr >> 13;
pTTL2[L3eIdx].UpdatePoolFence(UmdContext, false);
PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
L2TableAdr + L2eIdx * GMM_AUX_L2e_SIZE,
L2e.Value);
//initialize all L1e with invalid entries
for(i = 0; i < (uint32_t)GMM_AUX_L1_SIZE(pGmmGlobalContext); i++)
{
PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
L1TableAdr + i * sizeof(uint64_t),
InvalidEntry);
}
}
}
}
GMM_DPF(GFXDBG_CRITICAL, "Mapping surface: GPUVA=0x%016llx Size=0x%08x Aux_GPUVA=0x%016llx", StartAdr, BaseSize, CCS$Adr);
for(TileAdr = StartAdr; TileAdr < EndAdr; TileAdr += (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)),
CCS$Adr += (pGmmGlobalContext->GetSkuTable().FtrLinearCCS ?
(!WA16K ? GMM_BYTES(256) : GMM_BYTES(64)) :
0))
{
GMM_GFX_SIZE_T L1eIdx = GMM_L1_ENTRY_IDX(AUXTT, TileAdr, pGmmGlobalContext);
GMM_AUXTTL1e L1e = {0};
L1e.Value = PartialData;
L1e.Valid = 1;
CCS$Adr = (pGmmGlobalContext->GetSkuTable().FtrLinearCCS ? CCS$Adr :
__GetCCSCacheline(BaseResInfo, BaseAdr, AuxResInfo, AuxVA, TileAdr - BaseAdr));
if(!WA16K)
{
__GMM_ASSERT((CCS$Adr & 0xFF) == 0x0);
__GMM_ASSERT(GFX_IS_ALIGNED(CCS$Adr, GMM_BYTES(256)));
__GMM_ASSERT(GFX_IS_ALIGNED(TileAdr, GMM_KBYTE(64)));
L1e.GfxAddress = CCS$Adr >> 8; /*********** 256B-aligned CCS adr *****/
}
else
{
L1e.Reserved2 = CCS$Adr >> 6; /*********** 2 lsbs of 64B-aligned CCS adr *****/
L1e.GfxAddress = CCS$Adr >> 8; /*********** 256B-aligned CCS adr *****/
}
//GMM_DPF(GFXDBG_CRITICAL, "Map | L1=0x%016llx[0x%016llx] L1e=[0x%016llx] | [ GPUVA=0x%016llx[0x%08x] Aux_GPUVA=0x%016llx [%s]", L1TableAdr + L1eIdx << 3, L1eIdx << 3, L1e.Value, TileAdr, TileAdr - StartAdr, CCS$Adr, L1e.Valid ? "V" : " ");
GmmLib::LastLevelTable *pL1Tbl = NULL;
pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx, NULL);
L1TableCPUAdr = pL1Tbl->GetCPUAddress();
if(DoNotWait)
{
//Sync update on CPU
((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].Value = L1e.Value;
//GMM_DPF(GFXDBG_CRITICAL, "Map | Table Entry: [0x%06x] L2Addr[0x%016llX] Value[0x%016llX] :: [0x%06x] L1Addr[0x%016llX] Value[0x%016llX] -> GPUVA: 0x%016llX[0x%06X] Aux_GPUVA: 0x%016llX [%s]\n", L2eIdx, &((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx], ((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx].L1GfxAddr << 13, L1eIdx, &((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx] /*L1TableAdr + L1eIdx * GMM_AUX_L1e_SIZE*/, L1e.Value, TileAdr, TileAdr - BaseAdr, ((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].GfxAddress << 6, ((GMM_AUXTTL1e *)L1CPUAdr)[L1eIdx].Valid ? "V" : " ");
}
else
{
pL1Tbl->UpdatePoolFence(UmdContext, false);
PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
L1TableAdr + L1eIdx * GMM_AUX_L1e_SIZE,
L1e.Value);
}
// Since we are mapping a non-null entry, no need to check whether
// L1 table is unused.
pL1Tbl->TrackTableUsage(AUXTT, true, TileAdr, false);
}
}
if(!DoNotWait)
{
PageTableMgr->TTCb.pfEpilogTranslationTable(
UmdContext->pCommandQueueHandle,
1);
}
}
LeaveCriticalSection(&TTLock);
return Status;
}
GMM_AUXTTL1e GmmLib::AuxTable::CreateAuxL1Data(GMM_RESOURCE_INFO *BaseResInfo)
{
GMM_FORMAT_ENTRY FormatInfo = pGmmGlobalContext->GetPlatformInfo().FormatTable[BaseResInfo->GetResourceFormat()];
GMM_AUXTTL1e L1ePartial = {0};
#define GMM_REGISTRY_UMD_PATH "SOFTWARE\\Intel\\IGFX\\GMM\\"
#define GMM_E2EC_OVERRIDEDEPTH16BPPTO12 "ForceYUV16To12BPP"
L1ePartial.Mode = BaseResInfo->GetResFlags().Info.RenderCompressed ? 0x1 : 0x0; //MC on VCS supports all compression modes,
//MC on Render pipe only 128B compr (until B-step)
//Recognize which .MC surfaces needs Render pipe access
if(pGmmGlobalContext->GetWaTable().WaLimit128BMediaCompr)
{
L1ePartial.Mode = 0x1; //Limit media compression to 128B (same as RC) on gen12LP A0
}
//L1ePartial.Lossy = 0; // when to set it
L1ePartial.TileMode = BaseResInfo->GetResFlags().Info.TiledYs ? 0 : 1;
L1ePartial.Format = FormatInfo.CompressionFormat.AuxL1eFormat;
L1ePartial.LumaChroma = GmmIsPlanar(BaseResInfo->GetResourceFormat());
if(pGmmGlobalContext->GetWaTable().WaUntypedBufferCompression && BaseResInfo->GetResourceType() == RESOURCE_BUFFER)
{
//Gen12LP WA to support untyped raw buffer compression on HDC ie MLC(machine-learning compression)
L1ePartial.TileMode = 0;
L1ePartial.Depth = 0x6;
L1ePartial.Format = GMM_E2ECOMP_FORMAT_RGBAFLOAT16;
}
__GMM_ASSERT(L1ePartial.Format > GMM_E2ECOMP_MIN_FORMAT && //Are we going to reuse 0x00 for uncompressed indication? CCS contains that info, but only known by HW
L1ePartial.Format <= GMM_E2ECOMP_MAX_FORMAT); //Could SW use it as surface-wide uncompressed state indicator? If so, remove teh assert (Need to make sure, all format encodings are correct)
if(BaseResInfo->GetResFlags().Info.RenderCompressed)
{
if(BaseResInfo->GetResourceType() != RESOURCE_BUFFER)
{
switch(FormatInfo.Element.BitsPer)
{
case 8:
L1ePartial.Depth = 0x4;
break;
case 16:
L1ePartial.Depth = 0x0;
break;
case 32:
L1ePartial.Depth = 0x5;
break;
case 64:
L1ePartial.Depth = 0x6;
break;
case 128:
L1ePartial.Depth = 0x7;
break;
default:
L1ePartial.Depth = 0x3;
}
}
}
else
{
switch(BaseResInfo->GetResourceFormat())
{
case GMM_FORMAT_P012:
case GMM_FORMAT_Y412:
case GMM_FORMAT_Y212: //which format encoding for Y212, Y412, P012?
L1ePartial.Depth = 0x2;
break;
case GMM_FORMAT_P010:
//case GMM_FORMAT_Y410:
case GMM_FORMAT_Y210: //which format encoding for Y210?
L1ePartial.Depth = 0x1;
break;
case GMM_FORMAT_P016: //per HAS, separate encoding than P010, but a comment says to use P010 in AuxTable?
case GMM_FORMAT_Y416:
case GMM_FORMAT_Y216:
L1ePartial.Depth = 0x0;
break;
default:
L1ePartial.Depth = 0x3; //For MC, bpp got from format encoding
}
if(L1ePartial.Format == GMM_E2ECOMP_FORMAT_R10G10B10A2_UNORM)
{
L1ePartial.Format = GMM_E2ECOMP_FORMAT_RGB10b;
}
}
return L1ePartial;
}
GMM_GFX_ADDRESS GMM_INLINE GmmLib::AuxTable::__GetCCSCacheline(GMM_RESOURCE_INFO *BaseResInfo, GMM_GFX_ADDRESS BaseAdr,
GMM_RESOURCE_INFO *AuxResInfo, GMM_GFX_ADDRESS AuxVA, GMM_GFX_SIZE_T AdrOffset)
{
GMM_GFX_ADDRESS CCSChunkAdr = 0xFFFFFFF0;
uint32_t x = 0, y = 0;
uint32_t i = 0, j = 0;
uint32_t CCSXTile = 0, CCSYTile = 0;
GMM_UNREFERENCED_PARAMETER(BaseAdr);
bool BaseIsYF = BaseResInfo->GetResFlags().Info.TiledYf ? true : false;
uint32_t BasePitchInTiles = BaseResInfo->GetRenderPitchTiles();
//Find YF/YS TileId <x,y> for given main surface 16K-chunk
//and CCS$Id <i,j> corresponding to main's <x,y>
AdrOffset >>= 14; //AdrOffset must be 16K-aligned chunk, since mapping unit is 4 YF pages
if(BaseIsYF)
{
uint32_t PitchIn4YF = BasePitchInTiles / 4; //Base Pitch is physically padded to 4x1 YF width
i = static_cast<uint32_t>(AdrOffset % PitchIn4YF);
j = static_cast<uint32_t>(AdrOffset / PitchIn4YF);
}
else if(BasePitchInTiles != 0) //TileYs
{
x = static_cast<uint32_t>(AdrOffset >> 2); //YS-tile count
y = x / BasePitchInTiles; //YS- tile id <x,y>
x = x % BasePitchInTiles;
i = 2 * x;
j = 2 * y;
switch(AdrOffset % 4) //YS : XYXY [XYXY YF] ie 2x2 16K-units in Y-major
{
case 0:
break;
case 1:
j++;
break;
case 2:
i++;
break;
case 3:
i++;
j++;
break;
}
}
//Compute CCS$ address for <i,j>
CCSXTile = (i >= 8) ? i / 8 : 0; //8x8 CLs make one CCS Tile; get TileOffset
CCSYTile = (j >= 8) ? j / 8 : 0;
i %= 8;
j %= 8;
uint32_t AuxPitchInTiles = AuxResInfo ? AuxResInfo->GetRenderPitchTiles() : BaseResInfo->GetRenderAuxPitchTiles();
CCSChunkAdr = AuxVA + ((CCSXTile + CCSYTile * AuxPitchInTiles) * GMM_KBYTE(4)) + (8 * GMM_BYTES(64) * i) + (GMM_BYTES(64) * j);
return CCSChunkAdr;
}
#endif /*!__GMM_KMD__*/

730
Source/GmmLib/TranslationTable/GmmPageTableMgr.cpp Normal file
View File

@ -0,0 +1,730 @@
/*==============================================================================
Copyright(c) 2019 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files(the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and / or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
Description: UMD-TT manager (manages both TR-TT and AUX-TT in user mode space)
============================================================================*/
#include "Internal/Common/GmmLibInc.h"
#include "External/Common/GmmPageTableMgr.h"
#include "../TranslationTable/GmmUmdTranslationTable.h"
#include "External/Common/GmmClientContext.h"
#if defined(__linux__)
#include "Internal/Linux/GmmResourceInfoLinInt.h"
#endif
#define ENTER_CRITICAL_SECTION \
if(AuxTTObj) \
{ \
EnterCriticalSection(&PoolLock); \
}
#define EXIT_CRITICAL_SECTION \
if(AuxTTObj) \
{ \
LeaveCriticalSection(&PoolLock); \
}
#if defined(__linux__)
GMM_STATUS GmmLib::__GmmDeviceAlloc(GmmClientContext * pClientContext,
GMM_DEVICE_CALLBACKS_INT *pDeviceCbInt,
GMM_DEVICE_ALLOC * pAlloc)
{
GMM_CLIENT ClientType;
GMM_DDI_ALLOCATE Alloc = {0};
int err;
GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
__GMM_ASSERTPTR(GmmCheckForNullDevCbPfn(ClientType, pDeviceCbInt, GMM_DEV_CB_ALLOC), GMM_INVALIDPARAM);
if(GmmCheckForNullDevCbPfn(ClientType, pDeviceCbInt, GMM_DEV_CB_ALLOC))
{
Alloc.size = pAlloc->Size;
Alloc.alignment = pAlloc->Alignment;
err = GmmDeviceCallback(ClientType, pDeviceCbInt, &Alloc);
if(err)
{
return GMM_OUT_OF_MEMORY;
}
pAlloc->GfxVA = Alloc.gfxAddr;
pAlloc->CPUVA = Alloc.gfxAddr;
pAlloc->Handle = (HANDLE)Alloc.bo;
}
return GMM_SUCCESS;
}
GMM_STATUS GmmLib::__GmmDeviceDealloc(GMM_CLIENT ClientType,
GMM_DEVICE_CALLBACKS_INT *DeviceCb,
GMM_DEVICE_DEALLOC * pDealloc)
{
GMM_DDI_DEALLOCATE DeAlloc = {0};
int err = 0;
__GMM_ASSERTPTR(GmmCheckForNullDevCbPfn(ClientType, DeviceCb, GMM_DEV_CB_DEALLOC), GMM_INVALIDPARAM);
if(GmmCheckForNullDevCbPfn(ClientType, DeviceCb, GMM_DEV_CB_DEALLOC))
{
DeAlloc.bo = pDealloc->Handle;
err = GmmDeviceCallback(ClientType, DeviceCb, &DeAlloc);
}
return (err == 0) ? GMM_SUCCESS : GMM_ERROR;
}
#endif
//=============================================================================
//
// Function: __AllocateNodePool
//
// Desc: Allocates (always resident SVM) memory for new Pool node, and updates PageTableMgr object
//
// Parameters:
// AddrAlignment: Pool allocation address alignment
//
// Returns:
// S_OK on success,
//-----------------------------------------------------------------------------
GmmLib::GMM_PAGETABLEPool *GmmLib::GmmPageTableMgr::__AllocateNodePool(uint32_t AddrAlignment, GmmLib::POOL_TYPE Type)
{
GMM_STATUS Status = GMM_SUCCESS;
GMM_RESOURCE_INFO *pGmmResInfo = NULL;
GMM_PAGETABLEPool *pTTPool = NULL;
HANDLE PoolHnd = 0;
GMM_CLIENT ClientType;
GMM_DEVICE_ALLOC Alloc = {0};
ENTER_CRITICAL_SECTION
//Allocate pool, sized PAGETABLE_POOL_MAX_NODES pages, assignable to TR/Aux L1/L2 tables
//SVM allocation, always resident
Alloc.Size = PAGETABLE_POOL_SIZE;
Alloc.Alignment = AddrAlignment;
Alloc.hCsr = hCsr;
Status = __GmmDeviceAlloc(pClientContext, &DeviceCbInt, &Alloc);
if(Status != GMM_SUCCESS)
{
__GMM_ASSERT(0);
EXIT_CRITICAL_SECTION
return NULL;
}
PoolHnd = Alloc.Handle;
pGmmResInfo = (GMM_RESOURCE_INFO *)Alloc.Priv;
pTTPool = new GMM_PAGETABLEPool(PoolHnd, pGmmResInfo, Alloc.GfxVA, Alloc.CPUVA, Type);
__GMM_ASSERTPTR(pTTPool, NULL);
if(pTTPool)
{
if(pPool)
{
NumNodePoolElements++;
if(Type == POOL_TYPE_TRTTL2) // TRTT-L2 not 1st node in Pool LinkedList, place it at beginning
{
pPool = pPool->InsertInListAtBegin(pTTPool);
}
else
{
pTTPool = pPool->InsertInList(pTTPool);
}
}
else
{
NumNodePoolElements = 1;
pPool = pTTPool;
}
}
else
{
Status = GMM_OUT_OF_MEMORY;
}
EXIT_CRITICAL_SECTION
return (Status == GMM_SUCCESS) ? pTTPool : NULL;
}
//=============================================================================
//
// Function: __ReleaseUnusedPool
//
// Desc: Frees up unused PageTablePools once residency limit is hit
//
// Parameters:
// UmdContext: pointer to caller thread's context (containing BBHandle/Fence info)
//
//-----------------------------------------------------------------------------
void GmmLib::GmmPageTableMgr::__ReleaseUnusedPool(GMM_UMD_SYNCCONTEXT *UmdContext)
{
GMM_STATUS Status = GMM_SUCCESS;
GMM_GFX_SIZE_T PoolSizeToFree = {0};
GMM_GFX_SIZE_T FreedSize = {0};
GmmLib::GMM_PAGETABLEPool *Pool = NULL, *PrevPool = NULL;
uint32_t i = 0;
GMM_CLIENT ClientType;
GMM_DEVICE_DEALLOC Dealloc;
GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
ENTER_CRITICAL_SECTION
if(pPool->__IsUnusedTRTTPoolOverLimit(&PoolSizeToFree))
{
for(i = 0; i < NumNodePoolElements && FreedSize < PoolSizeToFree; i++)
{
Pool = (PrevPool) ? PrevPool->GetNextPool() : pPool;
if(Pool->IsPoolInUse(UmdContext ? SyncInfo(UmdContext->BBFenceObj, UmdContext->BBLastFence) : SyncInfo()))
{
PrevPool = Pool;
continue;
}
if(GmmCheckForNullDevCbPfn(ClientType, &DeviceCbInt, GMM_DEV_CB_WAIT_FROM_CPU))
{
GMM_DDI_WAITFORSYNCHRONIZATIONOBJECTFROMCPU Wait = {0};
Wait.bo = Pool->GetPoolHandle();
GmmDeviceCallback(ClientType, &DeviceCbInt, &Wait);
}
Dealloc.Handle = Pool->GetPoolHandle();
Dealloc.GfxVA = Pool->GetGfxAddress();
Dealloc.Priv = Pool->GetGmmResInfo();
Dealloc.hCsr = hCsr;
Status = __GmmDeviceDealloc(ClientType, &DeviceCbInt, &Dealloc);
__GMM_ASSERT(GMM_SUCCESS == Status);
if(PrevPool)
{
PrevPool->GetNextPool() = Pool->GetNextPool();
}
else
{
pPool = Pool->GetNextPool();
}
delete Pool;
FreedSize += PAGETABLE_POOL_SIZE;
}
}
EXIT_CRITICAL_SECTION
}
//=============================================================================
//
// Function: __GetFreePoolNode
//
// Desc: Finds free node within existing PageTablePool(s), if no such node found,
// allocates new PageTablePool. Caller should update Pool Node usage
//
// Parameters:
// FreePoolNodeIdx: pointer to return Pool's free Node index
// PoolType: AuxTT_L1/L2 pool
//
// Returns:
// PageTablePool element and FreePoolNodeIdx that should be used for L2/L1 assignment
// NULL, if no free node exists and new pool allocation failed
//-----------------------------------------------------------------------------
GmmLib::GMM_PAGETABLEPool *GmmLib::GmmPageTableMgr::__GetFreePoolNode(uint32_t *FreePoolNodeIdx, POOL_TYPE PoolType)
{
uint32_t PoolNode = -1, i = 0, j = 0, DWdivisor = 1, IdxMultiplier = 1;
bool PoolNodeFound = false, TRTTPool = false;
ENTER_CRITICAL_SECTION
GmmLib::GMM_PAGETABLEPool *Pool = pPool;
Pool = (PoolType == POOL_TYPE_TRTTL2) ? Pool : //1st pool reserved for TRTT-L2, since TRTT-L2 pruning not supported yet,
(Pool ? Pool->GetNextPool() : NULL); //other pools can be TR-L1/Aux-L1/Aux-L2 (and support dynamic pruning)
TRTTPool = (PoolType == POOL_TYPE_TRTTL2 || PoolType == POOL_TYPE_TRTTL1) ? true : false;
DWdivisor = TRTTPool ? 8 * sizeof(uint32_t) : (PoolType == POOL_TYPE_AUXTTL2) ? 8 * sizeof(uint32_t) * AUX_L2TABLE_SIZE_IN_POOLNODES : 8 * sizeof(uint32_t) * AUX_L1TABLE_SIZE_IN_POOLNODES;
IdxMultiplier = TRTTPool ? 1 : (PoolType == POOL_TYPE_AUXTTL2) ? AUX_L2TABLE_SIZE_IN_POOLNODES : AUX_L1TABLE_SIZE_IN_POOLNODES;
//Scan existing PageTablePools for free pool node
for(i = (PoolType == POOL_TYPE_TRTTL2) ? 0 : 1; Pool && i < NumNodePoolElements; i++)
{
if(Pool->GetNumFreeNode() > 0 && Pool->GetPoolType() == PoolType)
{
PoolNodeFound = true;
*FreePoolNodeIdx = 0;
for(; j < PAGETABLE_POOL_MAX_NODES / DWdivisor; j++)
{
if(_BitScanForward((uint32_t *)&PoolNode, (uint32_t) ~(Pool->GetNodeUsageAtIndex(j)))) // Get LSB that has value 0
{
*FreePoolNodeIdx += PoolNode * IdxMultiplier;
PoolNodeFound = true;
break;
}
PoolNodeFound = false;
*FreePoolNodeIdx += DWdivisor; //DWORD size in bits
}
}
if(PoolNodeFound)
{
__GMM_ASSERT(Pool->GetPoolType() == PoolType);
EXIT_CRITICAL_SECTION
return Pool;
}
Pool = Pool->GetNextPool();
}
//No free pool node, allocate new
if(!PoolNodeFound)
{
GMM_PAGETABLEPool *Pool = NULL;
if(Pool = __AllocateNodePool(IdxMultiplier * PAGE_SIZE, PoolType))
{
__GMM_ASSERT(Pool->GetPoolType() == PoolType);
*FreePoolNodeIdx = 0;
EXIT_CRITICAL_SECTION
return Pool;
}
}
EXIT_CRITICAL_SECTION
return NULL;
}
/**********************************************************************************
** Class GmmPageTableMgr functions **
***********************************************************************************/
/////////////////////////////////////////////////////////////////////////////////////
/// Instantiates GmmPageTableMgr, allocating memory for root-tables, copies provided
/// device-callback function pointers
///
/// @param[in] DeviceCb: pointer sharing device-callback function pointers
/// @param[in] TTFlags: Flags specifying which PageTables are required by client
/// @return GmmPageTableMgr*
/////////////////////////////////////////////////////////////////////////////////////
GmmLib::GmmPageTableMgr::GmmPageTableMgr(GMM_DEVICE_CALLBACKS_INT *DeviceCB, uint32_t TTFlags, GmmClientContext *pClientContextIn)
: GmmPageTableMgr()
{
GMM_PAGETABLE_MGR *ptr = NULL;
GMM_STATUS status = GMM_SUCCESS;
GMM_CLIENT ClientType;
if(pClientContextIn)
{
ClientType = pClientContextIn->GetClientType();
}
else
{
goto ERROR_CASE;
}
// this is needed if there is an error case and destructor gets called on ptr
this->pClientContext = pClientContextIn;
// Currently coping the code below to GMMOldAPi.cpp for backward compatible.
// Any changes here should be copied there.
//Initialize PageTableMgr further, only if PageTable creation succeeded
try
{
ptr = new GmmPageTableMgr();
ptr->pClientContext = pClientContextIn;
memcpy(&ptr->DeviceCbInt, DeviceCB, sizeof(GMM_DEVICE_CALLBACKS_INT));
if(pGmmGlobalContext->GetSkuTable().FtrE2ECompression)
{
__GMM_ASSERT(TTFlags & AUXTT); //Aux-TT is mandatory
ptr->AuxTTObj = new AuxTable();
if(!ptr->AuxTTObj)
{
goto ERROR_CASE;
}
ptr->AuxTTObj->PageTableMgr = ptr;
ptr->AuxTTObj->pClientContext = pClientContextIn;
status = ptr->AuxTTObj->AllocateL3Table(8 * PAGE_SIZE, 8 * PAGE_SIZE);
if(status != GMM_SUCCESS)
{
InitializeCriticalSection(&(ptr->PoolLock));
goto ERROR_CASE;
}
}
}
catch(...)
{
__GMM_ASSERT(false);
if(ptr && (AuxTTObj))
{
InitializeCriticalSection(&(ptr->PoolLock));
}
goto ERROR_CASE;
}
if(status == GMM_SUCCESS && !(AuxTTObj))
{
if(ptr->AuxTTObj)
{
ptr->AuxTTObj->PageTableMgr = this;
}
*this = *ptr;
//Don't initialize PoolLock until any of AuxTable object created
if(ptr->AuxTTObj )
{
InitializeCriticalSection(&PoolLock);
}
//Delete temporary ptr, but don't release allocated PageTable Obj.
ptr->AuxTTObj = NULL;
}
ERROR_CASE:
delete ptr;
ptr = NULL;
}
/////////////////////////////////////////////////////////////////////////////////////
/// Returns Root-table address for Aux-table
///
/// @return GMM_GFX_ADDRESS if Aux-Table was created; NULL otherwise
/////////////////////////////////////////////////////////////////////////////////////
GMM_GFX_ADDRESS GmmLib::GmmPageTableMgr::GetAuxL3TableAddr()
{
return AuxTTObj ? AuxTTObj->GetL3Address() : NULL;
}
/////////////////////////////////////////////////////////////////////////////////////
/// Queues commands to initialize Aux-Table registers in the HW context image
///
/// @param[in] initialBBHandle: pointer to BatchBuffer for queuing commands
/// @param[in] engType: specifes engine on which the context would run
/// @return GMM_SUCCESS if queuing succeeded; GMM_ERROR otherwise
/////////////////////////////////////////////////////////////////////////////////////
GMM_STATUS GmmLib::GmmPageTableMgr::InitContextAuxTableRegister(HANDLE CmdQHandle, GMM_ENGINE_TYPE engType)
{
GMM_GFX_ADDRESS MaskedL3GfxAddress = 0ULL;
GMM_UNREFERENCED_PARAMETER(engType);
//Check FtrE2ECompression = 1
if(pGmmGlobalContext->GetSkuTable().FtrE2ECompression && AuxTTObj != NULL)
{
EnterCriticalSection(&AuxTTObj->TTLock);
if(CmdQHandle)
{
//engType = ENGINE_TYPE_RCS; //use correct offset based on engType (once per-eng offsets known)
uint64_t RegOffset = 0, L3AdrReg = 0;
GET_L3ADROFFSET(0, L3AdrReg);
RegOffset = (L3AdrReg + sizeof(uint32_t));
RegOffset = L3AdrReg | (RegOffset << 0x20);
MaskedL3GfxAddress = AuxTTObj->GetL3Address();
//TTCb.pfPrologTranslationTable(CmdQHandle); //MI_FLUSH, TLBInv not required since its called during context-init
TTCb.pfWriteL3Adr(CmdQHandle, MaskedL3GfxAddress, RegOffset);
GMM_DPF(GFXDBG_CRITICAL, "AuxTT Map Address: GPUVA=0x%016llX\n", MaskedL3GfxAddress);
//TTCb.pfEpilogTranslationTable(CmdQHandle, 0);
AuxTTObj->GetRegisterStatus() = 0;
}
else
{
__GMM_ASSERT(false);
LeaveCriticalSection(&AuxTTObj->TTLock);
return GMM_INVALIDPARAM;
}
LeaveCriticalSection(&AuxTTObj->TTLock);
}
return GMM_SUCCESS;
}
/////////////////////////////////////////////////////////////////////////////////////
/// Updates the Aux-PageTables, for given base resource, with appropriate mappings
///
/// @param[in] Details of AuxTable update request
/// @return GMM_STATUS
/////////////////////////////////////////////////////////////////////////////////////
GMM_STATUS GmmLib::GmmPageTableMgr::UpdateAuxTable(const GMM_DDI_UPDATEAUXTABLE *UpdateReq)
{
if(GetAuxL3TableAddr() == NULL)
{
GMM_ASSERTDPF(0, "Invalid AuxTable update request, AuxTable is not initialized");
return GMM_INVALIDPARAM;
}
if(!((UpdateReq->BaseResInfo->GetResFlags().Info.RenderCompressed ||
UpdateReq->BaseResInfo->GetResFlags().Info.MediaCompressed) &&
((!UpdateReq->AuxResInfo && UpdateReq->BaseResInfo->GetResFlags().Gpu.UnifiedAuxSurface) ||
(UpdateReq->AuxResInfo && UpdateReq->AuxResInfo->GetResFlags().Gpu.CCS))))
/*(UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource ||
UpdateReq->BaseResInfo->GetResFlags().Gpu.Depth) */
//Allow Separate Aux for Depth/TR/MSAA/others?
{
GMM_ASSERTDPF(0, "Invalid AuxTable update request");
return GMM_INVALIDPARAM;
}
if(UpdateReq->Map && !(!UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource || (UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource && UpdateReq->UmdContext && UpdateReq->UmdContext->pCommandQueueHandle)))
{
//GMM_DPF_CRITICAL("TiledResources must Gpu-update AuxTable, proceeding with CPU-update...");
//Allowing CPU-update if requested so..
if(!UpdateReq->DoNotWait)
{
return GMM_INVALIDPARAM;
}
}
ENTER_CRITICAL_SECTION
if(UpdateReq->Map)
{
//Get AuxL1e data (other than CCS-adr) from main surface
uint64_t PartialL1e = AuxTTObj->CreateAuxL1Data(UpdateReq->BaseResInfo).Value;
GMM_STATUS Status = GMM_SUCCESS;
if(UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource)
{
//Aux-TT is sparsely updated, for TRs, upon change in mapping state ie
// null->non-null must be mapped
// non-null->null invalidated on AuxTT
uint8_t CpuUpdate = UpdateReq->DoNotWait || !(UpdateReq->UmdContext && UpdateReq->UmdContext->pCommandQueueHandle);
GMM_GFX_ADDRESS AuxVA = UpdateReq->AuxSurfVA;
if(UpdateReq->BaseResInfo->GetResFlags().Gpu.UnifiedAuxSurface)
{
GMM_UNIFIED_AUX_TYPE AuxType = GMM_AUX_CCS;
AuxType = (UpdateReq->BaseResInfo->GetResFlags().Gpu.Depth && UpdateReq->BaseResInfo->GetResFlags().Gpu.CCS) ? GMM_AUX_ZCS : AuxType;
AuxVA = UpdateReq->BaseGpuVA + GmmResGetAuxSurfaceOffset(UpdateReq->BaseResInfo, AuxType);
}
}
else
{
GMM_GFX_ADDRESS AuxVA = {0};
GMM_GFX_ADDRESS UVAuxVA = {0};
GMM_GFX_SIZE_T YPlaneSize = 0;
uint32_t MaxPlanes = 1;
if(!UpdateReq->AuxResInfo && UpdateReq->BaseResInfo->GetResFlags().Gpu.UnifiedAuxSurface)
{
GMM_UNIFIED_AUX_TYPE AuxType = GMM_AUX_CCS;
AuxType = (UpdateReq->BaseResInfo->GetResFlags().Gpu.Depth &&
UpdateReq->BaseResInfo->GetResFlags().Gpu.CCS) ?
GMM_AUX_ZCS :
AuxType;
AuxVA = UpdateReq->BaseGpuVA + GmmResGetAuxSurfaceOffset(UpdateReq->BaseResInfo, AuxType);
//For UV Packed, Gen12 e2e compr supported formats have 2 planes per surface
//Each has distinct Aux surface, Y-plane/UV-plane must be mapped to respective Y/UV Aux surface
if(GmmIsPlanar(UpdateReq->BaseResInfo->GetResourceFormat()))
{
GMM_REQ_OFFSET_INFO ReqInfo = {0};
ReqInfo.Plane = GMM_PLANE_U;
ReqInfo.ReqRender = 1;
MaxPlanes = 2;
UpdateReq->BaseResInfo->GetOffset(ReqInfo);
YPlaneSize = ReqInfo.Render.Offset64;
UVAuxVA = UpdateReq->BaseGpuVA + GmmResGetAuxSurfaceOffset(UpdateReq->BaseResInfo, GMM_AUX_UV_CCS);
}
}
//Per-plane Aux-TT map called with per-plane base/Aux address/size
for(uint32_t i = 0; i < MaxPlanes; i++)
{
GMM_GFX_SIZE_T SurfSize = (MaxPlanes > 1 && UpdateReq->BaseResInfo->GetArraySize() > 1) ?
(UpdateReq->BaseResInfo->GetQPitchPlanar(GMM_NO_PLANE) * UpdateReq->BaseResInfo->GetRenderPitch()) :
UpdateReq->BaseResInfo->GetSizeMainSurface();
GMM_GFX_SIZE_T MapSize = (i == 0) ? ((MaxPlanes > 1) ? YPlaneSize : SurfSize) : SurfSize - YPlaneSize;
GMM_GFX_ADDRESS BaseSurfVA = (UpdateReq->AuxResInfo || i == 0) ? UpdateReq->BaseGpuVA :
UpdateReq->BaseGpuVA + YPlaneSize;
GMM_GFX_ADDRESS AuxSurfVA = (UpdateReq->AuxResInfo) ? UpdateReq->AuxSurfVA : (i > 0 ? UVAuxVA : AuxVA);
//Luma plane reset LumaChroma bit
((GMM_AUXTTL1e *)&PartialL1e)->LumaChroma = (i == 0) ? 0 : 1;
uint32_t ArrayEle = GFX_MAX(((MaxPlanes > 1) ?
UpdateReq->BaseResInfo->GetArraySize() :
1),
1);
for(uint32_t j = 0; j < ArrayEle; j++)
{
BaseSurfVA += ((j > 0) ? (UpdateReq->BaseResInfo->GetQPitchPlanar(GMM_PLANE_Y) * UpdateReq->BaseResInfo->GetRenderPitch()) : 0);
AuxSurfVA += (UpdateReq->AuxResInfo ?
((j > 0) ? (UpdateReq->AuxResInfo->GetQPitchPlanar(GMM_PLANE_Y) * UpdateReq->BaseResInfo->GetRenderPitch()) : 0) :
((j > 0) ? UpdateReq->BaseResInfo->GetAuxQPitch() : 0));
//(Flat mapping): Remove main/aux resInfo from params
Status = AuxTTObj->MapValidEntry(UpdateReq->UmdContext, BaseSurfVA, MapSize, UpdateReq->BaseResInfo,
AuxSurfVA, UpdateReq->AuxResInfo, PartialL1e, 1);
if(Status != GMM_SUCCESS)
{
GMM_ASSERTDPF(0, "Insufficient memory, free resources and try again");
EXIT_CRITICAL_SECTION
return Status;
}
}
}
}
}
else
{
//Invalidate all mappings for given main surface
AuxTTObj->InvalidateTable(UpdateReq->UmdContext, UpdateReq->BaseGpuVA, UpdateReq->BaseResInfo->GetSizeMainSurface(), UpdateReq->DoNotWait);
}
EXIT_CRITICAL_SECTION
return GMM_SUCCESS;
}
#if defined(__linux__) && !_WIN32
/////////////////////////////////////////////////////////////////////////////////////
/// Gets size of PageTable buffer object (BOs) list
///
/// @param[in] TTFlags: Flags specifying PageTable-type for which BO-count required
/// @return non-zero if BO list is created. Zero otherwise.
/////////////////////////////////////////////////////////////////////////////////////
int GmmLib::GmmPageTableMgr::GetNumOfPageTableBOs(uint8_t TTFlags)
{
int NumBO = 0;
__GMM_ASSERTPTR(TTFlags & AUXTT, 0);
ENTER_CRITICAL_SECTION
if(AuxTTObj && AuxTTObj->GetL3Handle())
NumBO++;
NumBO += NumNodePoolElements;
EXIT_CRITICAL_SECTION
return NumBO;
}
/////////////////////////////////////////////////////////////////////////////////////
/// Gets size of PageTable buffer object (BOs) list
///
/// @param[in] TTFlags: Flags specifying PageTable-type for which BO-count required
/// @param[in][out] BOList: pointer to memory where PageTable BO*(s) must be sent
/// @return non-zero if BO list is created. Zero otherwise.
/////////////////////////////////////////////////////////////////////////////////////
int GmmLib::GmmPageTableMgr::GetPageTableBOList(uint8_t TTFlags, void *BOList)
{
int NumBO = GetNumOfPageTableBOs(TTFlags);
HANDLE * Handles = (HANDLE *)BOList;
GmmLib::GMM_PAGETABLEPool *Pool;
__GMM_ASSERTPTR(TTFlags & AUXTT, 0);
__GMM_ASSERTPTR(BOList, 0);
__GMM_ASSERTPTR(NumBO, 0);
ENTER_CRITICAL_SECTION
if(AuxTTObj && AuxTTObj->GetL3Handle())
Handles[0] = AuxTTObj->GetL3Handle();
Pool = pPool;
for(int i = 0; i < NumNodePoolElements; i++)
{
if(Pool)
{
Handles[i + 1] = Pool->GetPoolHandle();
Pool = Pool->GetNextPool();
}
}
EXIT_CRITICAL_SECTION
return NumBO;
}
#endif
/////////////////////////////////////////////////////////////////////////////////////
/// Releases GmmPageTableMgr, deleting root-tables and existing page-table pools
/////////////////////////////////////////////////////////////////////////////////////
GmmLib::GmmPageTableMgr::~GmmPageTableMgr()
{
GMM_CLIENT ClientType;
GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
if(pPool)
{
ENTER_CRITICAL_SECTION
pPool->__DestroyPageTablePool(&DeviceCbInt, hCsr);
NumNodePoolElements = 0;
EXIT_CRITICAL_SECTION
}
if(AuxTTObj)
{
DeleteCriticalSection(&PoolLock);
if(AuxTTObj)
{
if(AuxTTObj->NullL1Table)
{
delete AuxTTObj->NullL1Table;
}
if(AuxTTObj->NullL2Table)
{
delete AuxTTObj->NullL2Table;
}
AuxTTObj->DestroyL3Table();
delete AuxTTObj;
AuxTTObj = NULL;
}
}
}
/////////////////////////////////////////////////////////////////////////////////////
/// Instantiates and zeroes out GmmPageTableMgr
///
/// @return GmmPageTableMgr*
/////////////////////////////////////////////////////////////////////////////////////
GmmLib::GmmPageTableMgr::GmmPageTableMgr()
{
this->AuxTTObj = NULL;
this->pPool = NULL;
this->NumNodePoolElements = 0;
this->pClientContext = NULL;
this->hCsr = NULL;
memset(&DeviceCb, 0, sizeof(GMM_DEVICE_CALLBACKS));
memset(&DeviceCbInt, 0, sizeof(GMM_DEVICE_CALLBACKS_INT));
}

670
Source/GmmLib/TranslationTable/GmmUmdTranslationTable.cpp Normal file
View File

@ -0,0 +1,670 @@
/*==============================================================================
Copyright(c) 2019 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files(the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and / or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
Description: Contains functions of internal classes
(ie PageTablePool, PageTable, Table), that support
user mode page table management
============================================================================*/
#include "Internal/Common/GmmLibInc.h"
#include "../TranslationTable/GmmUmdTranslationTable.h"
#include "Internal/Common/Texture/GmmTextureCalc.h"
#if !defined(__GMM_KMD)
#if defined(__linux__)
#include "Internal/Linux/GmmResourceInfoLinInt.h"
#define _aligned_free(ptr) free(ptr)
#endif
//=============================================================================
//
// Function: AllocateL3Table
//
// Desc: Allocates (always resident SVM) memory for AUXTT\L3 Table, and updates AUXTT object
//
// Parameters:
// pAUXTT_Obj: per-device AUX-TT object. Contains AUXTT node info
//
// Returns:
// GMM_SUCCESS on success,
// GMM_INVALIDPARAM on invalid parameter(s)
// GMM_OUT_OF_MEMORY on memory allocation failure, failure to make resident
//-----------------------------------------------------------------------------
GMM_STATUS GmmLib::PageTable::AllocateL3Table(uint32_t L3TableSize, uint32_t L3AddrAlignment)
{
GMM_STATUS Status = GMM_SUCCESS;
GMM_DEVICE_ALLOC Alloc = {0};
__GMM_ASSERTPTR(PageTableMgr, GMM_INVALIDPARAM);
EnterCriticalSection(&TTLock);
Alloc.Size = L3TableSize;
Alloc.Alignment = L3AddrAlignment;
Alloc.hCsr = PageTableMgr->hCsr;
Status = __GmmDeviceAlloc(pClientContext, &PageTableMgr->DeviceCbInt, &Alloc);
if(Status != GMM_SUCCESS)
{
LeaveCriticalSection(&TTLock);
return Status;
}
TTL3.GfxAddress = GMM_GFX_ADDRESS_CANONIZE(Alloc.GfxVA);
TTL3.CPUAddress = Alloc.CPUVA;
TTL3.NeedRegisterUpdate = true;
TTL3.L3Handle = (HANDLE)(uintptr_t)Alloc.Handle;
TTL3.pGmmResInfo = (GMM_RESOURCE_INFO *)Alloc.Priv;
// Invalidate L3e's
for(int i = 0; i < (GMM_L3_SIZE(TTType)); i++)
{
//initialize L3e ie mark all entries with Null tile/invalid value
((GMM_AUXTTL3e *)TTL3.CPUAddress)[i].Value = 0;
}
LeaveCriticalSection(&TTLock);
return Status;
}
//=============================================================================
//
// Function: __IsUnusedTRTTPoolOverLimit
//
// Desc: Checks if unused TRTTPools have reached residency limit and must be freed.
//
// Parameters:
// pTRTT_Obj: per-device TT object. Contains TT node info
// OverLimitSize: Size in bytes that can be freed
//
// Returns:
// True, if unused TTPool reached max. residency limit
// False, otherwise
//-----------------------------------------------------------------------------
bool GmmLib::GmmPageTablePool::__IsUnusedTRTTPoolOverLimit(GMM_GFX_SIZE_T *OverLimitSize)
{
GMM_GFX_SIZE_T UnusedTrTTPoolSize = 0;
GmmLib::GMM_PAGETABLEPool *Pool = NULL;
Pool = this;
while(Pool)
{
if(Pool->NumFreeNodes == PAGETABLE_POOL_MAX_NODES)
{
UnusedTrTTPoolSize += PAGETABLE_POOL_SIZE;
}
Pool = Pool->NextPool;
}
*OverLimitSize = (UnusedTrTTPoolSize > PAGETABLE_POOL_MAX_UNUSED_SIZE) ? (UnusedTrTTPoolSize - PAGETABLE_POOL_MAX_UNUSED_SIZE) : 0;
return (UnusedTrTTPoolSize > PAGETABLE_POOL_MAX_UNUSED_SIZE) ? true : false;
}
//=============================================================================
//
// Function: AllocateL1L2Table
//
// Desc: Assigns pages from AUXTTPool for L1/L2Table for translation of given TRVA
//
// Parameters:
// pAUXTT_Obj: per-device AUX-TT object. Contains AUXTT node info
// TileAddr: Tiled Resource Virtual address
//
// Returns:
// L2Table/L1Table Address
//-----------------------------------------------------------------------------
void GmmLib::PageTable::AllocateL1L2Table(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS *L1TableAdr, GMM_GFX_ADDRESS *L2TableAdr)
{
GMM_GFX_ADDRESS L3TableAdr = GMM_NO_TABLE;
uint32_t L3eIdx = static_cast<uint32_t>(GMM_L3_ENTRY_IDX(TTType, TileAddr));
uint32_t L2eIdx = static_cast<uint32_t>(GMM_L2_ENTRY_IDX(TTType, TileAddr));
GmmLib::LastLevelTable *pL1Tbl = NULL;
*L2TableAdr = GMM_NO_TABLE;
*L1TableAdr = GMM_NO_TABLE;
if(TTL3.L3Handle)
{
L3TableAdr = TTL3.GfxAddress;
}
else
{
//Should never hit -- L3Table is allocated during device creation
__GMM_ASSERT(false);
}
if(pTTL2[L3eIdx].GetPool())
{
GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
PoolElem = pTTL2[L3eIdx].GetPool();
*L2TableAdr = (PoolElem != NULL) ? PoolElem->GetGfxAddress() + (PAGE_SIZE * pTTL2[L3eIdx].GetNodeIdx()) : GMM_NO_TABLE;
}
else
{
uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
POOL_TYPE PoolType = POOL_TYPE_AUXTTL2;
PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType);
if(PoolElem)
{
pTTL2[L3eIdx] = MidLevelTable(PoolElem, PoolNodeIdx, PoolElem->GetNodeBBInfoAtIndex(PoolNodeIdx));
*L2TableAdr = PoolElem->GetGfxAddress() + PAGE_SIZE * PoolNodeIdx; //PoolNodeIdx must be multiple of 8 (Aux L2) and multiple of 2 (Aux L1)
ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, NodesPerTable)
}
}
pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx);
if(pL1Tbl)
{
GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
PoolElem = pL1Tbl->GetPool();
*L1TableAdr = (PoolElem != NULL) ? PoolElem->GetGfxAddress() + (PAGE_SIZE * pL1Tbl->GetNodeIdx()) : GMM_NO_TABLE;
}
else
{
//Allocate L1 Table
uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
POOL_TYPE PoolType = POOL_TYPE_AUXTTL1;
PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType); //Recognize if Aux-L1 being allocated
if(PoolElem)
{
pL1Tbl = new GmmLib::LastLevelTable(PoolElem, PoolNodeIdx, GMM_L1_SIZE_DWORD(TTType, pGmmGlobalContext), L2eIdx); //use Aux L1_Size_DWORD
if(pL1Tbl)
{
*L1TableAdr = PoolElem->GetGfxAddress() + PAGE_SIZE * PoolNodeIdx; //PoolNodeIdx should reflect 1 node per Tr-table and 2 nodes per AUX L1 TABLE
if(PoolNodeIdx != PAGETABLE_POOL_MAX_NODES)
{
uint32_t PerTableNodes = (TTType == AUXTT) ? AUX_L1TABLE_SIZE_IN_POOLNODES : 1;
ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, PerTableNodes)
}
pTTL2[L3eIdx].InsertInList(pL1Tbl);
}
}
}
}
//=============================================================================
//
// Function: AllocateDummyTables
//
// Desc: Assigns pages from AUXTTPool for Dummy L1/L2Table
//
// Parameters:
// L2Table - Ptr to initiatize dummy L2Table
// L1Table - Ptr to initiatize dummy L1Table
//
// Returns:
// L2Table/L1Tables
//-----------------------------------------------------------------------------
void GmmLib::PageTable::AllocateDummyTables(GmmLib::Table **L2Table, GmmLib::Table **L1Table)
{
GMM_GFX_ADDRESS L3TableAdr = GMM_NO_TABLE;
GmmLib::LastLevelTable *pL1Tbl = NULL;
if(TTL3.L3Handle)
{
L3TableAdr = TTL3.GfxAddress;
}
else
{
//Should never hit -- L3Table is allocated during device creation
__GMM_ASSERT(false);
}
//Allocate dummy L2Table
{
uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
POOL_TYPE PoolType = POOL_TYPE_AUXTTL2;
PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType);
if(PoolElem)
{
*L2Table = new GmmLib::MidLevelTable(PoolElem, PoolNodeIdx, PoolElem->GetNodeBBInfoAtIndex(PoolNodeIdx));
ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, NodesPerTable)
}
}
//Allocate dummy L1Table
{
uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
POOL_TYPE PoolType = POOL_TYPE_AUXTTL1;
PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType); //Recognize if Aux-L1 being allocated
if(PoolElem)
{
*L1Table = new GmmLib::LastLevelTable(PoolElem, PoolNodeIdx, GMM_L1_SIZE_DWORD(TTType, pGmmGlobalContext), 0); //use Aux L1_Size_DWORD
if(*L1Table)
{
if(PoolNodeIdx != PAGETABLE_POOL_MAX_NODES)
{
uint32_t PerTableNodes = (TTType == AUXTT) ? AUX_L1TABLE_SIZE_IN_POOLNODES : 1;
ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, PerTableNodes)
}
}
}
}
}
//=============================================================================
//
// Function: GetL1L2TableAddr
//
// Desc: For given tile address, returns L1/L2 Table address if the table exists
//
// Parameters:
// pAUXTT_Obj: per-device AUX-TT object. Contains AXUTT node info
// TileAddr: Tiled Resource Virtual address
//
// Returns:
// L2Table/L1Table Address
//-----------------------------------------------------------------------------
void GmmLib::PageTable::GetL1L2TableAddr(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS *L1TableAdr, GMM_GFX_ADDRESS *L2TableAdr)
{
GMM_GFX_SIZE_T L3eIdx, L2eIdx, L1eIdx;
GMM_GFX_ADDRESS L3TableAdr = GMM_NO_TABLE;
*L2TableAdr = GMM_NO_TABLE;
*L1TableAdr = GMM_NO_TABLE;
L3eIdx = GMM_L3_ENTRY_IDX(TTType, TileAddr);
L2eIdx = GMM_L2_ENTRY_IDX(TTType, TileAddr);
L1eIdx = GMM_L1_ENTRY_IDX(TTType, TileAddr, pGmmGlobalContext);
__GMM_ASSERT(TTL3.L3Handle);
L3TableAdr = TTL3.GfxAddress;
if(pTTL2[L3eIdx].GetPool())
{
GmmLib::GMM_PAGETABLEPool *Pool = NULL;
GmmLib::LastLevelTable * pL1Tbl = NULL;
Pool = pTTL2[L3eIdx].GetPool();
if(Pool)
{
__GMM_ASSERT(Pool->GetNumFreeNode() != PAGETABLE_POOL_MAX_NODES);
__GMM_ASSERT(pTTL2[L3eIdx].GetNodeIdx() < PAGETABLE_POOL_MAX_NODES);
__GMM_ASSERT(Pool->GetNodeUsageAtIndex(pTTL2[L3eIdx].GetNodeIdx() / (32 * NodesPerTable)) != 0);
*L2TableAdr = Pool->GetGfxAddress() + PAGE_SIZE * (pTTL2[L3eIdx].GetNodeIdx());
}
pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx);
if(pL1Tbl && pL1Tbl->GetPool())
{
Pool = NULL;
Pool = pL1Tbl->GetPool();
if(Pool)
{
uint32_t PerTableNodes = (TTType == AUXTT) ? AUX_L1TABLE_SIZE_IN_POOLNODES : 1;
__GMM_ASSERT(Pool->GetNumFreeNode() != PAGETABLE_POOL_MAX_NODES);
__GMM_ASSERT(pL1Tbl->GetNodeIdx() < PAGETABLE_POOL_MAX_NODES);
__GMM_ASSERT(Pool->GetNodeUsageAtIndex(pL1Tbl->GetNodeIdx() / (32 * PerTableNodes)) != 0);
*L1TableAdr = Pool->GetGfxAddress() + PAGE_SIZE * (pL1Tbl->GetNodeIdx());
}
}
}
}
//=============================================================================
//
// Function: GetMappingType
//
// Desc: For given gfx address and size, returns MappingType (null/non-null or
// Valid/Invalid) of GfxVA and first gfx address have reverse mapping
//
/// @param[in] GfxVA: Gfx Address whose mapping type is being queried
/// @param[in] Size: Size of interested Gfx address range
/// @param[out] LastAddr : 1st Gfx Address having reverse mapping type
//
/// @return 1/0 : for non-null/valid vs null/invalid mapping
//-----------------------------------------------------------------------------
uint8_t GmmLib::PageTable::GetMappingType(GMM_GFX_ADDRESS GfxVA, GMM_GFX_SIZE_T Size, GMM_GFX_ADDRESS &LastAddr)
{
GMM_GFX_SIZE_T L3eIdx, L2eIdx, L1eIdx, L1EntrySize;
uint8_t MapType = 0; //true for non-null, false for null mapped
bool bFoundLastVA = false, bTerminate = false;
GMM_GFX_ADDRESS TileAddr = GfxVA;
L3eIdx = GMM_L3_ENTRY_IDX(TTType, GfxVA);
L2eIdx = GMM_L2_ENTRY_IDX(TTType, GfxVA);
L1eIdx = GMM_L1_ENTRY_IDX(TTType, GfxVA, pGmmGlobalContext);
L1EntrySize = !WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16);
EnterCriticalSection(&TTLock);
__GMM_ASSERT(TTL3.L3Handle);
#define GET_NEXT_L1TABLE(L1eIdx, L2eIdx, L3eIdx) \
{ \
L1eIdx = 0; \
L2eIdx++; \
if(L2eIdx == (GMM_L2_SIZE(TTType))) \
{ \
L2eIdx = 0; \
L3eIdx++; \
if(L3eIdx == (GMM_L3_SIZE(TTType))) \
{ \
bTerminate = true; \
} \
} \
}
#define GET_NEXT_L2TABLE(L1eIdx, L2eIdx, L3eIdx) \
{ \
L1eIdx = L2eIdx = 0; \
L3eIdx++; \
if(L3eIdx == (GMM_L3_SIZE(TTType))) \
{ \
bTerminate = true; \
} \
}
while(!(bFoundLastVA || bTerminate) && (TileAddr < GfxVA + Size))
{
if(pTTL2[L3eIdx].GetPool())
{
GmmLib::LastLevelTable *pL1Tbl = NULL;
pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx);
if(pL1Tbl && pL1Tbl->GetPool())
{
uint32_t LastBit = 0;
uint32_t i = static_cast<uint32_t>(L1eIdx) / 32;
while(!bFoundLastVA && i < (uint32_t)(GMM_L1_SIZE_DWORD(TTType, pGmmGlobalContext)))
{
uint32_t UsageDW = pL1Tbl->GetUsedEntries()[i++];
uint32_t BitNum = 31;
if(GfxVA == TileAddr)
{
BitNum = L1eIdx % 32;
MapType = ((UsageDW & __BIT(BitNum)) ? 0x1 : 0x0); //true for non-null, false for null mapped
UsageDW = (!MapType) ? UsageDW : ~UsageDW;
UsageDW = ((uint64_t)UsageDW >> (BitNum + 1)) << (BitNum + 1); // clear lsb <= BitNum
}
else
{
UsageDW = (!MapType) ? UsageDW : ~UsageDW;
}
if(_BitScanForward((uint32_t *)&LastBit, UsageDW)) // Gets lsb > BitNum, having reverse mapType
{
bFoundLastVA = true;
uint32_t NumTiles = (GfxVA == TileAddr) ? (LastBit - BitNum) : LastBit;
LastAddr = TileAddr + NumTiles * L1EntrySize;
}
else
{
uint32_t NumTiles = (GfxVA == TileAddr) ? (32 - BitNum) : 32;
TileAddr += NumTiles * L1EntrySize;
}
}
if(!bFoundLastVA)
{
GET_NEXT_L1TABLE(L1eIdx, L2eIdx, L3eIdx);
}
}
else //L2Entry is NULL
{
if(MapType) //First hit null-map
{
LastAddr = TileAddr;
bFoundLastVA = true;
}
else
{
GMM_GFX_SIZE_T NumTiles = GMM_L1_SIZE(TTType, pGmmGlobalContext);
if(GfxVA == TileAddr)
{
MapType = false;
NumTiles -= L1eIdx;
}
TileAddr += NumTiles * L1EntrySize;
GET_NEXT_L1TABLE(L1eIdx, L2eIdx, L3eIdx)
}
}
}
else //L3entry is NULL
{
if(MapType) //First hit null-map
{
LastAddr = TileAddr;
bFoundLastVA = true;
}
else
{
GMM_GFX_SIZE_T NumTiles = 0;
if(GfxVA == TileAddr)
{
MapType = false;
NumTiles = (GMM_L2_SIZE(TTType) - L2eIdx) * (GMM_L1_SIZE(TTType, pGmmGlobalContext) - L1eIdx);
}
else
{
NumTiles = ((GMM_L2_SIZE(TTType)) * (GMM_L1_SIZE(TTType, pGmmGlobalContext)));
}
TileAddr += NumTiles * L1EntrySize;
GET_NEXT_L2TABLE(L1eIdx, L2eIdx, L3eIdx)
}
}
}
if(!bFoundLastVA)
{
LastAddr = TileAddr;
}
LeaveCriticalSection(&TTLock);
return MapType;
}
//=============================================================================
//
// Function: TrackTableUsage
//
// Desc: For given tile address, updates Table Usage.If Table has all Nullmappings
// then its pool node can be unassigned
//
// Parameters:
// Type: Translation Table type (Aux)
// IsL1: Is called for L1table or L2 Table
// TileAddr: Tiled Resource Virtual address
// NullMapped: true if given tiled adr was null mapped, otherwise false
//
// Returns:
// true, if Table for given tile adr is all null mapped
// false,if Table does not exist or has non-null mapping
//-----------------------------------------------------------------------------
bool GmmLib::Table::TrackTableUsage(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr, bool NullMapped)
{
uint32_t EntryIdx;
uint32_t ElemNum = 0, BitNum = 0;
EntryIdx = IsL1 ? static_cast<uint32_t>(GMM_L1_ENTRY_IDX(Type, TileAdr, pGmmGlobalContext)) : static_cast<uint32_t>(GMM_L2_ENTRY_IDX(Type, TileAdr));
ElemNum = EntryIdx / (sizeof(UsedEntries[0]) * 8);
BitNum = EntryIdx % (sizeof(UsedEntries[0]) * 8);
if(NullMapped)
{
UsedEntries[ElemNum] &= ~(1 << BitNum);
}
else
{
UsedEntries[ElemNum] |= (1 << BitNum);
}
if(NullMapped)
{
int TableDWSize = IsL1 ? static_cast<int>(GMM_L1_SIZE_DWORD(Type, pGmmGlobalContext)) : static_cast<int>(GMM_L2_SIZE_DWORD(Type));
for(int i = 0; i < TableDWSize; i++)
{
if(UsedEntries[i])
{
return false;
}
}
}
return NullMapped ? true : false;
}
//=============================================================================
//
// Function: __IsTableNullMapped
//
// Desc: For given tile address, checks if given Table has all Nullmappings
// then its pool node can be unassigned
//
// Parameters:
// Type: Translation Table type (TR or Aux)
// IsL1: Is called for L1table or L2 Table
// TileAddr: Tiled Resource Virtual address
//
// Returns:
// true, if Table for given tile adr is all null mapped
// false,if Table has non-null mapping
//-----------------------------------------------------------------------------
bool GmmLib::Table::IsTableNullMapped(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr)
{
GMM_UNREFERENCED_PARAMETER(TileAdr);
int TableDWSize = IsL1 ? static_cast<int>(GMM_L1_SIZE_DWORD(Type, pGmmGlobalContext)) : static_cast<int>(GMM_L2_SIZE_DWORD(Type));
for(int i = 0; i < TableDWSize; i++)
{
if(UsedEntries[i])
{
return false;
}
}
return true;
}
//=============================================================================
//
// Function: __UpdatePoolFence
//
// Desc: Updates AUXTTPool's or Table's BBFenceObj/value with current BB fence
//
// Parameters:
// pAUXTT_Obj: per-device AUX-TT object. Contains AUXTT node info
// Table: L1/L2 table pointer
// L1Table: true for L1 Table, else false
// ClearNode: if true, Fence info is cleared for table
// false, Fence info is updated for table and pool
//-----------------------------------------------------------------------------
void GmmLib::Table::UpdatePoolFence(GMM_UMD_SYNCCONTEXT *UmdContext, bool ClearNode)
{
if(!ClearNode)
{
//update both node and pool with current fence/handle
PoolElem->GetPoolBBInfo().BBQueueHandle =
BBInfo.BBQueueHandle = UmdContext->BBFenceObj;
PoolElem->GetPoolBBInfo().BBFence =
BBInfo.BBFence = UmdContext->BBLastFence + 1; //Save incremented fence value, since DX does it during submission
}
else
{
//Clear node fence/handle
BBInfo.BBQueueHandle = 0;
BBInfo.BBFence = 0;
}
}
/////////////////////////////////////////////////////////////////////////////////////
/// Releases all PageTable Pool(s) existing in Linked List
///
/// @param[in] DeviceCallbacks pointer to device callbacks structure
/// @return GMM_STATUS
/////////////////////////////////////////////////////////////////////////////////////
GMM_STATUS GmmLib::GmmPageTablePool::__DestroyPageTablePool(void *DeviceCallbacks, HANDLE hCsr)
{
GMM_STATUS Status = GMM_SUCCESS;
GMM_DEVICE_CALLBACKS_INT *DeviceCb = static_cast<GMM_DEVICE_CALLBACKS_INT *>(DeviceCallbacks);
GMM_PAGETABLEPool *Node = this, *Next = NULL;
GMM_CLIENT ClientType;
GMM_DEVICE_DEALLOC Dealloc = {0};
//Evict/Free gpu Va is implictly done by OS when de-allocating
while(Node)
{
Next = Node->NextPool;
GET_GMM_CLIENT_TYPE(Node->pClientContext, ClientType);
Dealloc.Handle = Node->PoolHandle;
Dealloc.GfxVA = Node->PoolGfxAddress;
Dealloc.Priv = Node->pGmmResInfo;
Dealloc.hCsr = hCsr;
Status = __GmmDeviceDealloc(ClientType, DeviceCb, &Dealloc);
Node->PoolHandle = NULL;
Node->PoolGfxAddress = 0;
delete Node;
Node = Next;
}
return Status;
}
/////////////////////////////////////////////////////////////////////////////////////
/// Releases memory allocated to PageTable's Root-table
///
/// @return GMM_STATUS
/////////////////////////////////////////////////////////////////////////////////////
GMM_STATUS GmmLib::PageTable::DestroyL3Table()
{
GMM_STATUS Status = GMM_SUCCESS;
uint8_t hr = GMM_SUCCESS;
GMM_CLIENT ClientType;
GMM_DEVICE_DEALLOC Dealloc = {0};
GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
EnterCriticalSection(&TTLock);
if(TTL3.L3Handle)
{
Dealloc.Handle = TTL3.L3Handle;
Dealloc.GfxVA = TTL3.GfxAddress;
Dealloc.Priv = TTL3.pGmmResInfo;
Dealloc.hCsr = PageTableMgr->hCsr;
Status = __GmmDeviceDealloc(ClientType, &PageTableMgr->DeviceCbInt, &Dealloc);
TTL3.L3Handle = NULL;
TTL3.GfxAddress = 0;
TTL3.CPUAddress = 0;
}
LeaveCriticalSection(&TTLock);
return Status;
}
#endif //!__GMM_KMD

531
Source/GmmLib/TranslationTable/GmmUmdTranslationTable.h Normal file
View File

@ -0,0 +1,531 @@
/*==============================================================================
Copyright(c) 2019 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files(the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and / or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
Description: This file contains the class definitions for GmmPageTablePool
PageTable, and low-level Tables for user-mode PageTable management,
that is common for both Linux and Windows.
======================= end_copyright_notice ==================================*/
#pragma once
#include "External/Common/GmmPageTableMgr.h"
#ifdef __linux__
#include <pthread.h>
#include <string.h>
// Internal Linux version of MSDK APIs.
static inline void InitializeCriticalSection(pthread_mutex_t *mutex)
{
pthread_mutexattr_t Attr;
pthread_mutexattr_init(&Attr);
pthread_mutexattr_settype(&Attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(mutex, &Attr);
}
static inline void DeleteCriticalSection(pthread_mutex_t *mutex)
{
pthread_mutex_destroy(mutex);
}
static inline void EnterCriticalSection(pthread_mutex_t *mutex)
{
pthread_mutex_lock(mutex);
}
static inline void LeaveCriticalSection(pthread_mutex_t *mutex)
{
pthread_mutex_unlock(mutex);
}
#ifndef _BitScanForwardDefined
static inline int _BitScanForward(uint32_t *index, uint32_t mask)
{
int i;
i = ffsl(mask);
if(i > 0)
{
*index = (uint32_t)(i - 1);
return i;
}
return 0;
}
#endif
#endif
#define GMM_L1_SIZE(TTType, pGmmLibContext) GMM_AUX_L1_SIZE(pGmmLibContext)
#define GMM_L1_SIZE_DWORD(TTType, pGmmLibContext) GMM_AUX_L1_SIZE_DWORD(pGmmLibContext)
#define GMM_L2_SIZE(TTType) GMM_AUX_L2_SIZE
#define GMM_L2_SIZE_DWORD(TTType) GMM_AUX_L2_SIZE_DWORD
#define GMM_L3_SIZE(TTType) GMM_AUX_L3_SIZE
#define GMM_L1_ENTRY_IDX(TTType, GfxAddress, pGmmLibContext) GMM_AUX_L1_ENTRY_IDX((GfxAddress), (pGmmLibContext))
#define GMM_L2_ENTRY_IDX(TTType, GfxAddress) GMM_AUX_L2_ENTRY_IDX((GfxAddress))
#define GMM_L3_ENTRY_IDX(TTType, GfxAddress) GMM_AUX_L3_ENTRY_IDX((GfxAddress))
#ifdef GMM_ULT
#define GMM_L1_ENTRY_IDX_EXPORTED(TTType, GfxAddress, WA64KEx) GMM_AUX_L1_ENTRY_IDX_EXPORTED((GfxAddress), WA64KEx)
#endif
#ifdef __cplusplus
#include "External/Common/GmmMemAllocator.hpp"
//HW provides single-set of TR/Aux-TT registers for non-privileged programming
//Engine-specific offsets are HW-updated with programmed values.
#define GET_L3ADROFFSET(TRTT, L3AdrOffset) \
L3AdrOffset = 0x4200;
#define ASSIGN_POOLNODE(Pool, NodeIdx, PerTableNodes) { \
(Pool)->GetNodeUsageAtIndex((NodeIdx) / (32 *(PerTableNodes))) |= __BIT(((NodeIdx) / (PerTableNodes)) % 32); \
(Pool)->GetNodeBBInfoAtIndex((NodeIdx)/ (PerTableNodes)) = SyncInfo(); \
(Pool)->GetNumFreeNode() -= (PerTableNodes); \
}
#define DEASSIGN_POOLNODE(PageTableMgr, UmdContext, Pool, NodeIdx, PerTableNodes) { \
(Pool)->GetNodeUsageAtIndex((NodeIdx) / (32 * (PerTableNodes))) &= ~__BIT(((NodeIdx) / (PerTableNodes)) % 32 ); \
(Pool)->GetNumFreeNode() += (PerTableNodes); \
if((Pool)->GetNumFreeNode() == PAGETABLE_POOL_MAX_NODES) { \
PageTableMgr->__ReleaseUnusedPool((UmdContext)); \
} \
}
namespace GmmLib
{
#define PAGETABLE_POOL_MAX_NODES 512 //Max. number of L2/L1 tables pool contains
#define PAGETABLE_POOL_SIZE_IN_DWORD PAGETABLE_POOL_MAX_NODES / 32
#define PAGETABLE_POOL_SIZE PAGETABLE_POOL_MAX_NODES * PAGE_SIZE //Pool for L2/L1 table allocation
#define AUX_L2TABLE_SIZE_IN_POOLNODES 8 //Aux L2 is 32KB
#define AUX_L1TABLE_SIZE_IN_POOLNODES 2 //Aux L1 is 8KB
#define PAGETABLE_POOL_MAX_UNUSED_SIZE GMM_MBYTE(16) //Max. size of unused pool, driver keeps resident
//////////////////////////////////////////////////////////////////////////////////////////////
/// Contains functions and members for GmmPageTablePool.
/// PageTablePool is a Linked-list, provides common location for both Aux TT and TR-TT pages
/// Separate NodePool (linked-list element) kept for each PoolType, for cleaner management in
/// per-table size
/////////////////////////////////////////////////////////////////////////////////////////////
class GmmPageTablePool
{
private:
//PageTablePool allocation descriptor
GMM_RESOURCE_INFO* pGmmResInfo;
HANDLE PoolHandle;
GMM_GFX_ADDRESS PoolGfxAddress;
GMM_GFX_ADDRESS CPUAddress; //LMEM-cpuvisible adr
POOL_TYPE PoolType; //Separate Node-pools for TR-L2, TR-L1, Aux-L2, Aux-L1 usages-
//PageTablePool usage descriptors
int NumFreeNodes; //has value {0 to Pool_Max_nodes}
uint32_t* NodeUsage; //destined node state (updated during node assignment and removed based on destined state of L1/L2 Table
//that used the pool node)
//Aux-Pool node-usage tracked at every eighth/second node(for L2 vs L1)
//ie 1b per node for TR-table, 1b per 8-nodes for Aux-L2table, 1b per 2-nodes for AuxL1-table
//array size= POOL_SIZE_IN_DWORD for TR, =POOL_SIZE_IN_DWORD/8 for AuxL2, POOL_SIZE_IN_DWORD/2 for AuxL1
SyncInfo* NodeBBInfo; //BB info for pending Gpu usage of each pool node
//array of size MaxPoolNodes for TR, =MaxPoolNodes / 8 for Aux, MaxPoolNodes / 2 for AuxL1
SyncInfo PoolBBInfo; //BB info for Gpu usage of the Pool (most recent of pool node BB info)
GmmPageTablePool* NextPool; //Next node-Pool in the LinkedList
GmmClientContext *pClientContext; ///< ClientContext of the client creating this Object
public:
GmmPageTablePool() :
pGmmResInfo(NULL),
PoolHandle(),
PoolGfxAddress(0x0),
CPUAddress(0x0),
PoolType(POOL_TYPE_TRTTL1),
NumFreeNodes(PAGETABLE_POOL_MAX_NODES),
NodeUsage(NULL),
NodeBBInfo(NULL),
PoolBBInfo(),
NextPool(NULL),
pClientContext(NULL)
{
}
GmmPageTablePool(HANDLE hAlloc, GMM_RESOURCE_INFO* pGmmRes, GMM_GFX_ADDRESS SysMem, POOL_TYPE Type) :
GmmPageTablePool()
{
PoolHandle = hAlloc;
pGmmResInfo = pGmmRes;
PoolGfxAddress = SysMem;
CPUAddress = PoolGfxAddress;
NextPool = NULL;
NumFreeNodes = PAGETABLE_POOL_MAX_NODES;
PoolType = Type;
int DwordPoolSize = (Type == POOL_TYPE_AUXTTL1) ? PAGETABLE_POOL_SIZE_IN_DWORD / AUX_L1TABLE_SIZE_IN_POOLNODES
: (Type == POOL_TYPE_AUXTTL2) ? PAGETABLE_POOL_SIZE_IN_DWORD / AUX_L2TABLE_SIZE_IN_POOLNODES
: PAGETABLE_POOL_SIZE_IN_DWORD;
NodeUsage = new uint32_t[DwordPoolSize]();
NodeBBInfo = new SyncInfo[DwordPoolSize * 32]();
if (pGmmResInfo)
{
pClientContext = pGmmResInfo->GetGmmClientContext();
}
}
GmmPageTablePool(HANDLE hAlloc, GMM_RESOURCE_INFO* pGmmRes, GMM_GFX_ADDRESS GfxAdr, GMM_GFX_ADDRESS CPUAdr, POOL_TYPE Type) :
GmmPageTablePool(hAlloc, pGmmRes, GfxAdr, Type)
{
CPUAddress = (CPUAdr != GfxAdr) ? CPUAdr : GfxAdr;
}
~GmmPageTablePool()
{
delete[] NodeUsage;
delete[] NodeBBInfo;
}
GmmPageTablePool* InsertInList(GmmPageTablePool* NewNode)
{
GmmPageTablePool *Node = this;
while (Node->NextPool)
{
Node = Node->NextPool;
}
Node->NextPool = NewNode;
return Node->NextPool;
}
GmmPageTablePool* InsertInListAtBegin(GmmPageTablePool* NewNode)
{
GmmPageTablePool *Node = this;
NewNode->NextPool = Node;
return NewNode;
}
GmmPageTablePool* &GetNextPool() { return NextPool; }
HANDLE& GetPoolHandle() { return PoolHandle; }
POOL_TYPE& GetPoolType() { return PoolType; }
int& GetNumFreeNode() { return NumFreeNodes; }
SyncInfo& GetPoolBBInfo() { return PoolBBInfo; }
uint32_t& GetNodeUsageAtIndex(int j) { return NodeUsage[j]; }
SyncInfo& GetNodeBBInfoAtIndex(int j) { return NodeBBInfo[j]; }
GMM_GFX_ADDRESS GetGfxAddress() { return PoolGfxAddress; }
GMM_GFX_ADDRESS GetCPUAddress() { return CPUAddress; }
GMM_RESOURCE_INFO* &GetGmmResInfo() { return pGmmResInfo; }
bool IsPoolInUse(SyncInfo BBInfo) {
if (NumFreeNodes < PAGETABLE_POOL_MAX_NODES ||
(PoolBBInfo.BBQueueHandle == BBInfo.BBQueueHandle &&
PoolBBInfo.BBFence == BBInfo.BBFence + 1)) //Pool will be used by next BB submission, freeing it will cause page fault
{
return true;
}
return false;
}
bool __IsUnusedTRTTPoolOverLimit(GMM_GFX_SIZE_T * OverLimitSize);
void ClearBBReference(void * BBQHandle);
GMM_STATUS __DestroyPageTablePool(void * DeviceCallbacks,HANDLE hCsr);
};
//////////////////////////////////////////////////////////////////////////////////////////////
/// Contains functions and members for Table.
/// Table defines basic building block for tables at different page-table levels
/////////////////////////////////////////////////////////////////////////////////////////////
class Table
{
protected:
GMM_PAGETABLEPool *PoolElem; //L2 Pool ptr different for L2Tables when Pool_nodes <512
int PoolNodeIdx; //pool node idx used for L2 Table
SyncInfo BBInfo; //BB Handle/fence using Table
uint32_t* UsedEntries; //Tracks which L1/L2 entries are being used
//array size GMM_L1_SIZE_DWORD(TT-type) for LastLevelTable, MidLeveltable(??)
//array of 1024/32=32 DWs for TR-table, 4096/32 =512 for Aux-Table
public:
Table() :
PoolElem(NULL),
PoolNodeIdx(),
BBInfo(),
UsedEntries(NULL)
{
}
int& GetNodeIdx() { return PoolNodeIdx; }
GmmPageTablePool* &GetPool() { return PoolElem; }
GMM_GFX_ADDRESS GetCPUAddress() { return (PoolElem->GetCPUAddress() + (PoolNodeIdx * PAGE_SIZE)); }
SyncInfo& GetBBInfo() { return BBInfo; }
uint32_t* &GetUsedEntries() { return UsedEntries; }
bool TrackTableUsage(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr, bool NullMapped);
bool IsTableNullMapped(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr);
void UpdatePoolFence(GMM_UMD_SYNCCONTEXT * UmdContext, bool ClearNode);
};
//////////////////////////////////////////////////////////////////////////////////////////////
/// Contains functions and members for LastLevelTable.
/// LastLevelTable defines leaf level tables in multi-level pageTable structure
/////////////////////////////////////////////////////////////////////////////////////////////
class LastLevelTable : public Table
{
private:
uint32_t L2eIdx;
LastLevelTable *pNext;
public:
LastLevelTable() : Table(),
L2eIdx() //Pass in Aux vs TR table's GMM_L2_SIZE and initialize L2eIdx?
{
pNext = NULL;
}
LastLevelTable(GMM_PAGETABLEPool* Elem, int NodeIdx, int DwordL1e, int L2eIndex)
: LastLevelTable()
{
PoolElem = Elem;
PoolNodeIdx = NodeIdx;
BBInfo = Elem->GetNodeBBInfoAtIndex(NodeIdx);
L2eIdx = L2eIndex;
pNext = NULL;
UsedEntries = new uint32_t[DwordL1e]();
}
~LastLevelTable()
{
delete[] UsedEntries;
}
int GetL2eIdx() {
return L2eIdx;
}
LastLevelTable* &Next() {
return pNext;
}
};
//////////////////////////////////////////////////////////////////////////////////////////////
/// Contains functions and members for MidLevelTable.
/// MidLevelTable defines secondary level tables in multi-level pageTable structure
/////////////////////////////////////////////////////////////////////////////////////////////
class MidLevelTable : public Table
{
private:
LastLevelTable *pTTL1; //linked list of L1 tables
public:
MidLevelTable() :Table()
{
pTTL1 = NULL;
}
MidLevelTable(GMM_PAGETABLEPool *Pool, int NodeIdx, SyncInfo Info) : MidLevelTable()
{
PoolElem = Pool;
BBInfo = Info;
PoolNodeIdx = NodeIdx;
}
~MidLevelTable()
{
if (pTTL1)
{
LastLevelTable* item = pTTL1;
while (item)
{
LastLevelTable* nextItem = item->Next();
delete item;
item = nextItem;
}
pTTL1 = NULL;
}
}
LastLevelTable* GetL1Table(GMM_GFX_SIZE_T L2eIdx, LastLevelTable** Prev = NULL)
{
LastLevelTable* pL1Tbl = pTTL1;
LastLevelTable* PrevL1Tbl = NULL;
while (pL1Tbl)
{
if (pL1Tbl->GetL2eIdx() == L2eIdx)
{
break;
}
PrevL1Tbl = pL1Tbl;
pL1Tbl = pL1Tbl->Next();
}
//if requested, save previous node in linked-list
if (Prev)
{
*Prev = PrevL1Tbl;
}
return pL1Tbl;
}
void InsertInList(LastLevelTable* pL1Tbl)
{
LastLevelTable* Prev = pTTL1;
//Insert at end
while (Prev && Prev->Next())
{
Prev = Prev->Next();
}
if (Prev)
{
Prev->Next() = pL1Tbl;
}
else
{
pTTL1 = pL1Tbl;
}
}
void DeleteFromList(LastLevelTable* pL1Tbl, LastLevelTable* PrevL1Tbl)
{
//Save next L1Table in list, before deleting current one
if (pL1Tbl)
{
if (PrevL1Tbl)
{
PrevL1Tbl->Next() = pL1Tbl->Next();
}
else
{
pTTL1 = pL1Tbl->Next();
}
delete pL1Tbl;
}
}
};
/////////////////////////////////////////////////////
/// Contains functions and members for PageTable.
/// PageTable defines multi-level pageTable
/////////////////////////////////////////////////////
class PageTable :
public GmmMemAllocator
{
protected:
const TT_TYPE TTType; //PageTable is AuxTT
const int NodesPerTable; //Aux L2/L3 has 32KB size, Aux L1 has 4KB -can't use as selector for PageTable is AuxTT
// 1 node for TR-table, 8 nodes for Aux-Table L2, 2 nodes for Aux-table L1
//Root Table structure
struct
{
GMM_RESOURCE_INFO* pGmmResInfo;
HANDLE L3Handle;
GMM_GFX_ADDRESS GfxAddress; //L3 Table Adr CPU equivalent GPU addr
GMM_GFX_ADDRESS CPUAddress; //LMEM-cpuvisible adr
bool NeedRegisterUpdate; //True @ L3 allocation, False when L3AdrRegWrite done
SyncInfo BBInfo;
} TTL3;
MidLevelTable* pTTL2; //array of L2-Tables
public:
#ifdef _WIN32
CRITICAL_SECTION TTLock; //synchronized access of PageTable obj
#elif defined __linux__
pthread_mutex_t TTLock;
#endif
GmmPageTableMgr* PageTableMgr;
GmmClientContext *pClientContext;
PageTable(int Size, int NumL3e, TT_TYPE flag) :
NodesPerTable(Size / PAGE_SIZE),
TTType(flag)
{
memset(&TTL3, 0, sizeof(TTL3));
InitializeCriticalSection(&TTLock);
pTTL2 = new MidLevelTable[NumL3e];
}
~PageTable()
{
delete[] pTTL2;
DeleteCriticalSection(&TTLock);
}
GMM_GFX_ADDRESS GetL3Address() { return TTL3.GfxAddress; }
bool &GetRegisterStatus() { return TTL3.NeedRegisterUpdate; }
GMM_STATUS AllocateL3Table(uint32_t L3TableSize, uint32_t L3AddrAlignment);
GMM_STATUS DestroyL3Table();
void AllocateL1L2Table(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS * L1TableAdr, GMM_GFX_ADDRESS * L2TableAdr);
void AllocateDummyTables(GmmLib::Table **L2Table, GmmLib::Table **L1Table);
void GetL1L2TableAddr(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS * L1TableAdr, GMM_GFX_ADDRESS* L2TableAdr);
uint8_t GetMappingType(GMM_GFX_ADDRESS GfxVA, GMM_GFX_SIZE_T Size, GMM_GFX_ADDRESS& LastAddr);
HANDLE GetL3Handle() { return TTL3.L3Handle; }
};
//////////////////////////////////////////////////////////////////////////////////////////////
/// Contains functions and members for AuxTable.
/// AuxTable defines PageTable for translating VA->AuxVA, ie defines page-walk to get address
/// of CCS-cacheline containing auxiliary data (compression tag, etc) for some resource
/////////////////////////////////////////////////////////////////////////////////////////////
class AuxTable : public PageTable
{
public:
const int L1Size;
Table* NullL2Table;
Table* NullL1Table;
GMM_GFX_ADDRESS NullCCSTile;
AuxTable() : PageTable(8 * PAGE_SIZE, GMM_AUX_L3_SIZE, TT_TYPE::AUXTT),
L1Size(2 * PAGE_SIZE)
{
NullL2Table = nullptr;
NullL1Table = nullptr;
NullCCSTile = 0;
}
GMM_STATUS InvalidateTable(GMM_UMD_SYNCCONTEXT * UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint8_t DoNotWait);
GMM_STATUS MapValidEntry(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T BaseSize,
GMM_RESOURCE_INFO* BaseResInfo, GMM_GFX_ADDRESS AuxVA, GMM_RESOURCE_INFO* AuxResInfo, uint64_t PartialData, uint8_t DoNotWait);
GMM_STATUS MapNullCCS(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint64_t PartialL1e, uint8_t DoNotWait);
GMM_AUXTTL1e CreateAuxL1Data(GMM_RESOURCE_INFO* BaseResInfo);
GMM_GFX_ADDRESS GMM_INLINE __GetCCSCacheline(GMM_RESOURCE_INFO* BaseResInfo, GMM_GFX_ADDRESS BaseAdr, GMM_RESOURCE_INFO* AuxResInfo,
GMM_GFX_ADDRESS AuxVA, GMM_GFX_SIZE_T AdrOffset);
};
typedef struct _GMM_DEVICE_ALLOC {
uint32_t Size;
uint32_t Alignment;
HANDLE Handle;
GMM_GFX_ADDRESS GfxVA;
GMM_GFX_ADDRESS CPUVA;
void * Priv;
HANDLE hCsr;
} GMM_DEVICE_ALLOC;
typedef struct _GMM_DEVICE_DEALLOC {
HANDLE Handle;
GMM_GFX_ADDRESS GfxVA;
void * Priv;
HANDLE hCsr;
} GMM_DEVICE_DEALLOC;
GMM_STATUS __GmmDeviceAlloc(GmmClientContext *pClientContext,
GMM_DEVICE_CALLBACKS_INT *pDeviceCbInt,
GMM_DEVICE_ALLOC *pAlloc);
GMM_STATUS __GmmDeviceDealloc(GMM_CLIENT ClientType,
GMM_DEVICE_CALLBACKS_INT *DeviceCb,
GMM_DEVICE_DEALLOC *pDealloc);
}
#endif // #ifdef __cplusplus

293
Source/GmmLib/ULT/CMakeLists.txt
View File

@ -1,141 +1,152 @@
# Copyright(c) 2017 Intel Corporation
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files(the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and / or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
set (EXE_NAME GMMULT)
set(GMMULT_HEADERS
GmmCachePolicyULT.h
GmmCommonULT.h
GmmGen10CachePolicyULT.h
GmmGen10ResourceULT.h
GmmGen11CachePolicyULT.h
GmmGen11ResourceULT.h
GmmGen12ResourceULT.h
GmmGen12CachePolicyULT.h
GmmGen9CachePolicyULT.h
GmmGen9ResourceULT.h
GmmResourceULT.h
stdafx.h
targetver.h
)
set(GMMULT_SOURCES
GmmCachePolicyULT.cpp
GmmCommonULT.cpp
GmmGen10CachePolicyULT.cpp
GmmGen10ResourceULT.cpp
GmmGen11CachePolicyULT.cpp
GmmGen12CachePolicyULT.cpp
GmmGen11ResourceULT.cpp
GmmGen12ResourceULT.cpp
GmmGen9CachePolicyULT.cpp
GmmGen9ResourceULT.cpp
GmmResourceCpuBltULT.cpp
GmmResourceULT.cpp
googletest/src/gtest-all.cc
GmmULT.cpp
)
source_group("Source Files\\Cache Policy" FILES
GmmCachePolicyULT.cpp
GmmGen9CachePolicyULT.cpp
GmmGen10CachePolicyULT.cpp
GmmGen11CachePolicyULT.cpp
GmmGen12CachePolicyULT.cpp
)
source_group("Source Files\\Resource" FILES
GmmGen10ResourceULT.cpp
GmmGen11ResourceULT.cpp
GmmGen12ResourceULT.cpp
GmmGen9ResourceULT.cpp
GmmResourceCpuBltULT.cpp
GmmResourceULT.cpp
)
source_group("Header Files\\Cache Policy" FILES
GmmCachePolicyULT.h
GmmGen10CachePolicyULT.h
GmmGen11CachePolicyULT.h
GmmGen12CachePolicyULT.h
GmmGen9CachePolicyULT.h
)
source_group("Header Files\\Resource" FILES
GmmGen12ResourceULT.h
GmmGen11ResourceULT.h
GmmGen10ResourceULT.h
GmmGen9ResourceULT.h
GmmResourceULT.h
)
source_group("gtest" FILES
googletest/gtest/gtest.h
googletest/src/gtest-all.cc
)
include_directories(BEFORE ./)
include_directories(BEFORE ${PROJECT_SOURCE_DIR})
include_directories(
googletest
googletest/gtest
${BS_DIR_INC}/umKmInc
${BS_DIR_INC}
${BS_DIR_GMMLIB}/inc
${BS_DIR_INC}/common
)
macro(GmmLibULTSetTargetConfig ultTarget)
if (TARGET ${ultTarget})
set_property(TARGET ${ultTarget} APPEND PROPERTY COMPILE_DEFINITIONS
$<$<CONFIG:Release>: _RELEASE>
$<$<CONFIG:ReleaseInternal>: _RELEASE_INTERNAL>
$<$<CONFIG:Debug>: _DEBUG>
)
endif()
endmacro()
add_executable(${EXE_NAME} ${GMMULT_HEADERS} ${GMMULT_SOURCES})
GmmLibULTSetTargetConfig(${EXE_NAME})
set_property(TARGET ${EXE_NAME} APPEND PROPERTY COMPILE_DEFINITIONS __GMM GMM_LIB_DLL __UMD)
if(NOT TARGET igfx_gmmumd_dll)
add_subdirectory("${BS_DIR_GMMLIB}" "${CMAKE_BINARY_DIR}/gmmlib/ult")
endif()
target_link_libraries(${EXE_NAME} igfx_gmmumd_dll)
target_link_libraries(${EXE_NAME}
pthread
dl
)
add_custom_target(Run_ULT ALL DEPENDS GMMULT)
add_custom_command(
TARGET Run_ULT
POST_BUILD
COMMAND echo running ULTs
COMMAND "${CMAKE_COMMAND}" -E env "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:igfx_gmmumd_dll>" ${CMAKE_CFG_INTDIR}/${EXE_NAME} --gtest_filter=CTest*
)
# Copyright(c) 2017 Intel Corporation
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files(the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and / or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
set (EXE_NAME GMMULT)
set(GMMULT_HEADERS
GmmCachePolicyULT.h
GmmCommonULT.h
GmmGen10CachePolicyULT.h
GmmGen10ResourceULT.h
GmmGen11CachePolicyULT.h
GmmGen11ResourceULT.h
GmmGen12ResourceULT.h
GmmGen12CachePolicyULT.h
GmmGen9CachePolicyULT.h
GmmGen9ResourceULT.h
GmmResourceULT.h
GmmAuxTableULT.h
stdafx.h
targetver.h
)
set(GMMULT_SOURCES
GmmCachePolicyULT.cpp
GmmCommonULT.cpp
GmmGen10CachePolicyULT.cpp
GmmGen10ResourceULT.cpp
GmmGen11CachePolicyULT.cpp
GmmGen12CachePolicyULT.cpp
GmmGen11ResourceULT.cpp
GmmGen12ResourceULT.cpp
GmmGen9CachePolicyULT.cpp
GmmGen9ResourceULT.cpp
GmmResourceCpuBltULT.cpp
GmmResourceULT.cpp
GmmAuxTableULT.cpp
googletest/src/gtest-all.cc
GmmULT.cpp
)
source_group("Source Files\\Cache Policy" FILES
GmmCachePolicyULT.cpp
GmmGen9CachePolicyULT.cpp
GmmGen10CachePolicyULT.cpp
GmmGen11CachePolicyULT.cpp
GmmGen12CachePolicyULT.cpp
)
source_group("Source Files\\Resource" FILES
GmmGen10ResourceULT.cpp
GmmGen11ResourceULT.cpp
GmmGen12ResourceULT.cpp
GmmGen9ResourceULT.cpp
GmmResourceCpuBltULT.cpp
GmmResourceULT.cpp
)
source_group("Source Files\\TranslationTable" FILES
GmmAuxTableULT.cpp
)
source_group("Header Files\\TranslationTable" FILES
GmmAuxTableULT.h
)
source_group("Header Files\\Cache Policy" FILES
GmmCachePolicyULT.h
GmmGen10CachePolicyULT.h
GmmGen11CachePolicyULT.h
GmmGen12CachePolicyULT.h
GmmGen9CachePolicyULT.h
)
source_group("Header Files\\Resource" FILES
GmmGen10ResourceULT.h
GmmGen11ResourceULT.h
GmmGen12ResourceULT.h
GmmGen9ResourceULT.h
GmmResourceULT.h
)
source_group("gtest" FILES
googletest/gtest/gtest.h
googletest/src/gtest-all.cc
)
include_directories(BEFORE ./)
include_directories(BEFORE ${PROJECT_SOURCE_DIR})
include_directories(
googletest
googletest/gtest
${BS_DIR_INC}/umKmInc
${BS_DIR_INC}
${BS_DIR_GMMLIB}/inc
${BS_DIR_INC}/common
)
macro(GmmLibULTSetTargetConfig ultTarget)
if (TARGET ${ultTarget})
set_property(TARGET ${ultTarget} APPEND PROPERTY COMPILE_DEFINITIONS
$<$<CONFIG:Release>: _RELEASE>
$<$<CONFIG:ReleaseInternal>: _RELEASE_INTERNAL>
$<$<CONFIG:Debug>: _DEBUG>
)
endif()
endmacro()
add_executable(${EXE_NAME} ${GMMULT_HEADERS} ${GMMULT_SOURCES})
GmmLibULTSetTargetConfig(${EXE_NAME})
set_property(TARGET ${EXE_NAME} APPEND PROPERTY COMPILE_DEFINITIONS __GMM GMM_LIB_DLL __UMD)
if(NOT TARGET igfx_gmmumd_dll)
add_subdirectory("${BS_DIR_GMMLIB}" "${CMAKE_BINARY_DIR}/gmmlib/ult")
endif()
target_link_libraries(${EXE_NAME} igfx_gmmumd_dll)
target_link_libraries(${EXE_NAME}
pthread
dl
)
add_custom_target(Run_ULT ALL DEPENDS GMMULT)
add_custom_command(
TARGET Run_ULT
POST_BUILD
COMMAND echo running ULTs
COMMAND "${CMAKE_COMMAND}" -E env "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:igfx_gmmumd_dll>" ${CMAKE_CFG_INTDIR}/${EXE_NAME} --gtest_filter=CTest*
)

265
Source/GmmLib/ULT/GmmAuxTableULT.cpp Normal file
View File

@ -0,0 +1,265 @@
/*==============================================================================
Copyright(c) 2019 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files(the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and / or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
============================================================================*/
#ifdef __linux__
#include "GmmAuxTableULT.h"
using namespace std;
using namespace GmmLib;
static GMM_DEVICE_CALLBACKS_INT DeviceCBInt;
CTestAuxTable::CTestAuxTable()
{
}
CTestAuxTable::~CTestAuxTable()
{
}
int CTestAuxTable::allocCB(void *bufMgr, size_t size, size_t alignment, void **bo, void **cpuAddr, uint64_t *gpuAddr)
{
if(bufMgr != (void *)0xdeadbeef)
return -1;
if(!bo || !cpuAddr || !gpuAddr)
return -2;
*cpuAddr = aligned_alloc(alignment, ALIGN(size, alignment));
if(!*cpuAddr)
return -3;
*bo = *cpuAddr;
*gpuAddr = (uint64_t)*cpuAddr;
return 0;
}
void CTestAuxTable::freeCB(void *bo)
{
ASSERT_TRUE(bo != NULL);
free(bo);
}
void CTestAuxTable::waitFromCpuCB(void *bo)
{
}
void CTestAuxTable::SetUpTestCase()
{
GfxPlatform.eProductFamily = IGFX_TIGERLAKE_LP;
GfxPlatform.eRenderCoreFamily = IGFX_GEN12_CORE;
DeviceCBInt.pBufMgr = (void *)0xdeadbeef;
DeviceCBInt.DevCbPtrs_.pfnAllocate = CTestAuxTable::allocCB;
DeviceCBInt.DevCbPtrs_.pfnDeallocate = CTestAuxTable::freeCB;
DeviceCBInt.DevCbPtrs_.pfnWaitFromCpu = CTestAuxTable::waitFromCpuCB;
if(GfxPlatform.eProductFamily == IGFX_UNKNOWN ||
GfxPlatform.eRenderCoreFamily == IGFX_UNKNOWN_CORE)
{
GfxPlatform.eProductFamily = IGFX_BROADWELL;
GfxPlatform.eRenderCoreFamily = IGFX_GEN8_CORE;
}
AllocateAdapterInfo();
if(pGfxAdapterInfo)
{
pGfxAdapterInfo->SkuTable.FtrE2ECompression = true;
pGfxAdapterInfo->SkuTable.FtrLinearCCS = true;
}
CommonULT::SetUpTestCase();
}
void CTestAuxTable::TearDownTestCase()
{
CommonULT::TearDownTestCase();
}
TEST_F(CTestAuxTable, TestUpdateAuxTableCompressedSurface)
{
GmmPageTableMgr *mgr = pGmmULTClientContext->CreatePageTblMgrObject(&DeviceCBInt, TT_TYPE::AUXTT);
ASSERT_TRUE(mgr != NULL);
Surface *surf = new Surface(7680, 4320);
ASSERT_TRUE(surf != NULL && surf->init());
GMM_DDI_UPDATEAUXTABLE updateReq = {0};
updateReq.BaseResInfo = surf->getGMMResourceInfo();
updateReq.BaseGpuVA = surf->getGfxAddress(GMM_PLANE_Y);
updateReq.Map = 1;
GMM_STATUS res = mgr->UpdateAuxTable(&updateReq);
ASSERT_TRUE(res == GMM_SUCCESS);
delete surf;
pGmmULTClientContext->DestroyPageTblMgrObject(mgr);
}
TEST_F(CTestAuxTable, DISABLED_TestUpdateAuxTableNonCompressedSurface)
{
GmmPageTableMgr *mgr = pGmmULTClientContext->CreatePageTblMgrObject(&DeviceCBInt, TT_TYPE::AUXTT);
ASSERT_TRUE(mgr != NULL);
Surface *surf = new Surface(7680, 4320, false);
ASSERT_TRUE(surf != NULL && surf->init());
GMM_DDI_UPDATEAUXTABLE updateReq = {0};
memset(&updateReq, 0, sizeof(GMM_DDI_UPDATEAUXTABLE));
updateReq.BaseResInfo = surf->getGMMResourceInfo();
updateReq.BaseGpuVA = surf->getGfxAddress(GMM_PLANE_Y);
updateReq.Map = 1;
GMM_STATUS res = mgr->UpdateAuxTable(&updateReq);
ASSERT_TRUE(res != GMM_SUCCESS);
delete surf;
pGmmULTClientContext->DestroyPageTblMgrObject(mgr);
}
TEST_F(CTestAuxTable, TestInvalidateAuxTable)
{
GmmPageTableMgr *mgr = pGmmULTClientContext->CreatePageTblMgrObject(&DeviceCBInt, TT_TYPE::AUXTT);
ASSERT_TRUE(mgr != NULL);
Surface *surf = new Surface(7680, 4320);
ASSERT_TRUE(surf != NULL && surf->init());
GMM_DDI_UPDATEAUXTABLE updateReq = {0};
updateReq.BaseResInfo = surf->getGMMResourceInfo();
updateReq.BaseGpuVA = surf->getGfxAddress(GMM_PLANE_Y);
updateReq.Map = 1;
GMM_STATUS res = mgr->UpdateAuxTable(&updateReq);
ASSERT_TRUE(res == GMM_SUCCESS);
memset(&updateReq, 0, sizeof(updateReq));
updateReq.BaseResInfo = surf->getGMMResourceInfo();
updateReq.Map = 0;
res = mgr->UpdateAuxTable(&updateReq);
ASSERT_TRUE(res == GMM_SUCCESS);
delete surf;
pGmmULTClientContext->DestroyPageTblMgrObject(mgr);
}
TEST_F(CTestAuxTable, DISABLED_TestUpdateAuxTableStress)
{
const int num_surf = 1000;
Surface * surfaces[num_surf];
Surface * surf;
int i;
GmmPageTableMgr *mgr = pGmmULTClientContext->CreatePageTblMgrObject(&DeviceCBInt, TT_TYPE::AUXTT);
ASSERT_TRUE(mgr != NULL);
for(i = 0; i < num_surf; i++)
{
surf = new Surface(7680, 4320);
surfaces[i] = surf;
ASSERT_TRUE(surf != NULL && surf->init());
GMM_DDI_UPDATEAUXTABLE updateReq = {0};
updateReq.BaseResInfo = surf->getGMMResourceInfo();
updateReq.BaseGpuVA = surf->getGfxAddress(GMM_PLANE_Y);
updateReq.Map = 1;
mgr->UpdateAuxTable(&updateReq);
}
for(i = 0; i < num_surf; i++)
{
surf = surfaces[i];
delete surf;
}
pGmmULTClientContext->DestroyPageTblMgrObject(mgr);
}
TEST_F(CTestAuxTable, TestAuxTableContent)
{
GmmPageTableMgr *mgr = pGmmULTClientContext->CreatePageTblMgrObject(&DeviceCBInt, TT_TYPE::AUXTT);
ASSERT_TRUE(mgr != NULL);
Surface *surf = new Surface(720, 480);
ASSERT_TRUE(surf != NULL && surf->init());
GMM_DDI_UPDATEAUXTABLE updateReq = {0};
updateReq.BaseResInfo = surf->getGMMResourceInfo();
updateReq.BaseGpuVA = surf->getGfxAddress(GMM_PLANE_Y);
updateReq.Map = 1;
GMM_STATUS res = mgr->UpdateAuxTable(&updateReq);
ASSERT_TRUE(res == GMM_SUCCESS);
Walker *ywalker = new Walker(surf->getGfxAddress(GMM_PLANE_Y),
surf->getAuxGfxAddress(GMM_AUX_CCS),
mgr->GetAuxL3TableAddr());
for(size_t i = 0; i < surf->getSurfaceSize(GMM_PLANE_Y); i++)
{
GMM_GFX_ADDRESS addr = surf->getGfxAddress(GMM_PLANE_Y) + i;
GMM_GFX_ADDRESS val = ywalker->walk(addr);
GMM_GFX_ADDRESS expected = ywalker->expected(addr);
ASSERT_EQ(expected, val);
}
Walker *uvwalker = new Walker(surf->getGfxAddress(GMM_PLANE_U),
surf->getAuxGfxAddress(GMM_AUX_UV_CCS),
mgr->GetAuxL3TableAddr());
for(size_t i = 0; i < surf->getSurfaceSize(GMM_PLANE_U); i++)
{
GMM_GFX_ADDRESS addr = surf->getGfxAddress(GMM_PLANE_U) + i;
GMM_GFX_ADDRESS val = uvwalker->walk(addr);
GMM_GFX_ADDRESS expected = uvwalker->expected(addr);
ASSERT_EQ(expected, val);
}
delete uvwalker;
delete ywalker;
delete surf;
pGmmULTClientContext->DestroyPageTblMgrObject(mgr);
}
#endif /* __linux__ */

252
Source/GmmLib/ULT/GmmAuxTableULT.h Normal file
View File

@ -0,0 +1,252 @@
/*==============================================================================
Copyright(c) 2019 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files(the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and / or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
============================================================================*/
#pragma once
#ifdef __linux__
#ifndef _ISOC11_SOURCE
#define _ISOC11_SOURCE 1
#endif
#include "GmmGen10ResourceULT.h"
#include <stdlib.h>
#include <malloc.h>
#ifndef ALIGN
#define ALIGN(v, a) (((v) + ((a)-1)) & ~((a)-1))
#endif
class CTestAuxTable : public CTestGen10Resource
{
public:
static void SetUpTestCase();
static void TearDownTestCase();
CTestAuxTable();
~CTestAuxTable();
static int allocCB(void *bufMgr, size_t size, size_t alignment, void **bo, void **cpuAddr, uint64_t *gpuAddr);
static void freeCB(void *bo);
static void waitFromCpuCB(void *bo);
class Surface
{
public:
Surface(unsigned int width, unsigned int height, bool mmc = true)
: mWidth(width), mHeight(height), mMMC(mmc), mResInfo(0), mBuf(0)
{
}
~Surface()
{
deinit();
}
bool init()
{
size_t size;
size_t alignment;
GMM_RESCREATE_PARAMS gmmParams = {};
gmmParams.Type = RESOURCE_2D;
gmmParams.Format = GMM_FORMAT_NV12;
gmmParams.BaseWidth = mWidth;
gmmParams.BaseHeight = mHeight;
gmmParams.Depth = 0x1;
gmmParams.ArraySize = 1;
gmmParams.Flags.Info.TiledY = 1;
gmmParams.Flags.Info.MediaCompressed = mMMC ? 1 : 0;
//gmmParams.Flags.Gpu.CCS = mmc ? 1 : 0;
gmmParams.Flags.Gpu.MMC = mMMC ? 1 : 0;
gmmParams.Flags.Gpu.Texture = 1;
gmmParams.Flags.Gpu.RenderTarget = 1;
gmmParams.Flags.Gpu.UnifiedAuxSurface = mMMC ? 1 : 0;
//gmmParams.Flags.Gpu.Depth = 1;
gmmParams.Flags.Gpu.Video = true;
mResInfo = pGmmULTClientContext->CreateResInfoObject(&gmmParams);
size = mResInfo->GetSizeSurface();
alignment = mResInfo->GetResFlags().Info.TiledYf ? GMM_KBYTE(16) : GMM_KBYTE(64);
mBuf = aligned_alloc(alignment, ALIGN(size, alignment));
if(!mResInfo || !mBuf)
return false;
mYBase = (GMM_GFX_ADDRESS)mBuf;
mUVBase = 0;
mAuxYBase = mYBase + mResInfo->GetUnifiedAuxSurfaceOffset(GMM_AUX_CCS);
mAuxUVBase = 0;
mYPlaneSize = mResInfo->GetSizeMainSurface();
if(pGmmULTClientContext->IsPlanar(mResInfo->GetResourceFormat()))
{
GMM_REQ_OFFSET_INFO ReqInfo = {0};
ReqInfo.Plane = GMM_PLANE_U;
ReqInfo.ReqRender = 1;
mResInfo->GetOffset(ReqInfo);
mYPlaneSize = ReqInfo.Render.Offset64;
mUVBase = mYBase + mYPlaneSize;
mAuxUVBase = mYBase + mResInfo->GetUnifiedAuxSurfaceOffset(GMM_AUX_UV_CCS);
}
mUVPlaneSize = mResInfo->GetSizeMainSurface() - mYPlaneSize;
return true;
}
void deinit()
{
if(mBuf)
{
free(mBuf);
mBuf = NULL;
}
if(mResInfo)
{
pGmmULTClientContext->DestroyResInfoObject(mResInfo);
mResInfo = NULL;
}
}
GMM_GFX_ADDRESS getGfxAddress(GMM_YUV_PLANE plane)
{
switch(plane)
{
case GMM_PLANE_Y:
return mYBase;
case GMM_PLANE_U:
case GMM_PLANE_V:
return mUVBase;
default:
throw;
}
}
GMM_GFX_ADDRESS getAuxGfxAddress(GMM_UNIFIED_AUX_TYPE auxType)
{
switch(auxType)
{
case GMM_AUX_CCS:
case GMM_AUX_Y_CCS:
return mAuxYBase;
case GMM_AUX_UV_CCS:
return mAuxUVBase;
default:
throw;
}
}
GMM_RESOURCE_INFO *getGMMResourceInfo()
{
return mResInfo;
}
size_t getSurfaceSize(GMM_YUV_PLANE plane)
{
switch(plane)
{
case GMM_PLANE_Y:
return mYPlaneSize;
case GMM_PLANE_U:
case GMM_PLANE_V:
return mUVPlaneSize;
default:
throw;
}
}
private:
unsigned int mWidth;
unsigned int mHeight;
bool mMMC;
GMM_RESOURCE_INFO *mResInfo;
void * mBuf;
GMM_GFX_ADDRESS mYBase;
GMM_GFX_ADDRESS mUVBase;
GMM_GFX_ADDRESS mAuxYBase;
GMM_GFX_ADDRESS mAuxUVBase;
size_t mYPlaneSize;
size_t mUVPlaneSize;
};
class Walker
{
public:
Walker(GMM_GFX_ADDRESS mainBase, GMM_GFX_ADDRESS auxBase,
GMM_GFX_ADDRESS l3Base)
{
mMainBase = mainBase;
mAuxBase = (auxBase >> 6) << 6;
mL3Base = (uint64_t *)l3Base;
}
GMM_GFX_ADDRESS expected(GMM_GFX_ADDRESS addr)
{
uint8_t Is64KChunk = (const_cast<WA_TABLE &>(pGfxAdapterInfo->WaTable).WaAuxTable16KGranular) ? 0 : 1;
uint32_t count = (addr - mMainBase) / (Is64KChunk ? GMM_KBYTE(64) : GMM_KBYTE(16));
return mAuxBase + (Is64KChunk ? 256 : 64) * count;
}
GMM_GFX_ADDRESS walk(GMM_GFX_ADDRESS addr)
{
uint64_t mask = (const_cast<WA_TABLE &>(pGfxAdapterInfo->WaTable).WaAuxTable16KGranular) ? 0x0000ffffffffffc0 : 0x0000ffffffffff00;
uint32_t idx = l3Index(addr);
uint64_t *l2Base = (uint64_t *)((mL3Base[idx] >> 15) << 15);
idx = l2Index(addr);
uint64_t *l1Base = (uint64_t *)((l2Base[idx] >> 13) << 13);
idx = l1Index(addr);
uint64_t auxAddr = (uint64_t)(l1Base[idx] & mask);
return auxAddr;
}
public:
static inline uint32_t l3Index(GMM_GFX_ADDRESS addr)
{
return GMM_AUX_L3_ENTRY_IDX(addr);
}
static inline uint32_t l2Index(GMM_GFX_ADDRESS addr)
{
return GMM_AUX_L2_ENTRY_IDX(addr);
}
static inline uint32_t l1Index(GMM_GFX_ADDRESS addr)
{
return GMM_AUX_L1_ENTRY_IDX_EXPORTED(addr, !(const_cast<WA_TABLE &>(pGfxAdapterInfo->WaTable).WaAuxTable16KGranular));
}
private:
GMM_GFX_ADDRESS mMainBase;
GMM_GFX_ADDRESS mAuxBase;
uint64_t * mL3Base;
};
};
#endif /* __linux__ */

1
Source/GmmLib/ULT/GmmCommonULT.cpp
View File

@ -25,6 +25,7 @@ OTHER DEALINGS IN THE SOFTWARE.
#include <dlfcn.h>
#endif
ADAPTER_INFO * CommonULT::pGfxAdapterInfo = NULL;
PLATFORM CommonULT::GfxPlatform = {};
GMM_CLIENT_CONTEXT *CommonULT::pGmmULTClientContext = NULL;

1
Source/GmmLib/ULT/stdafx.h
View File

@ -59,6 +59,7 @@ extern "C" {
#include "../inc/External/Common/GmmInfoExt.h"
#include "../inc/External/Common/GmmInfo.h"
#include "../inc/External/Common/GmmClientContext.h"
#include "../inc/External/Common/GmmPageTableMgr.h"
#include "../inc/External/Common/GmmLibDll.h"
#include "../inc/External/Common/GmmLibDllName.h"

14
Source/GmmLib/inc/External/Common/GmmClientContext.h vendored
View File

@ -122,7 +122,7 @@ namespace GmmLib
GMM_VIRTUAL GMM_RESOURCE_INFO* GMM_STDCALL CopyResInfoObject(GMM_RESOURCE_INFO *pSrcRes);
GMM_VIRTUAL void GMM_STDCALL ResMemcpy(void *pDst, void *pSrc);
GMM_VIRTUAL void GMM_STDCALL DestroyResInfoObject(GMM_RESOURCE_INFO *pResInfo);
#ifdef GMM_LIB_DLL
/* ResourceInfo and PageTableMgr Create and Destroy APIs with Client provided Memory Allocators */
GMM_VIRTUAL GMM_RESOURCE_INFO* GMM_STDCALL CreateResInfoObject(GMM_RESCREATE_PARAMS *pCreateParams,
@ -130,6 +130,18 @@ namespace GmmLib
GMM_VIRTUAL void GMM_STDCALL DestroyResInfoObject(GMM_RESOURCE_INFO *pResInfo,
GmmClientAllocationCallbacks *pAllocCbs);
#endif
/* PageTableMgr Creation and Destroy API's */
GMM_VIRTUAL GMM_PAGETABLE_MGR* GMM_STDCALL CreatePageTblMgrObject(GMM_DEVICE_CALLBACKS_INT *pDevCb, uint32_t TTFlags);
/* PageTableMgr Creation and Destroy API's */
GMM_VIRTUAL void GMM_STDCALL DestroyPageTblMgrObject(GMM_PAGETABLE_MGR *pPageTableMgr);
GMM_VIRTUAL GMM_PAGETABLE_MGR* GMM_STDCALL CreatePageTblMgrObject(
GMM_DEVICE_CALLBACKS_INT* pDevCb,
uint32_t TTFlags,
GmmClientAllocationCallbacks* pAllocCbs);
GMM_VIRTUAL void GMM_STDCALL DestroyPageTblMgrObject(GMM_PAGETABLE_MGR* pPageTableMgr,
GmmClientAllocationCallbacks* pAllocCbs);
};
}

4
Source/GmmLib/inc/External/Common/GmmCommonExt.h vendored
View File

@ -72,6 +72,10 @@ extern const SWIZZLE_DESCRIPTOR INTEL_64KB_UNDEFINED_64_128bpp;
#define GMM_NO_FENCE_REG 0xDEADBEEF
#define GMM_MAX_DISPLAYS 3
#if defined __linux__
typedef void* HANDLE;
#endif
//===========================================================================
// typedef:
// GMM_GFX_ADDRESS/etc.

150
Source/GmmLib/inc/External/Common/GmmHw.h vendored Normal file
View File

@ -0,0 +1,150 @@
/*==============================================================================
Copyright(c) 2019 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files(the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and / or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
============================================================================*/
#pragma once
#include "gfxmacro.h"
////////////////////// Auxiliary Translation Table definitions//////////////////////////////////////////
//===========================================================================
// typedef:
// GMM_AUXTTL3e
//
// Description:
// Struct for Auxiliary Translation-Table L3 entry
//--------------------------------------------------------------------------
typedef union GMM_AUXTTL3e_REC
{
struct {
uint64_t Valid : 1;
uint64_t Reserved0 : 14;
uint64_t L2GfxAddr : 33;
uint64_t Reserved1 : 16;
};
uint64_t Value;
} GMM_AUXTTL3e;
C_ASSERT(sizeof(GMM_AUXTTL3e) == 8);
// Get the L2GfxAddr bit field as a full L2 graphics address
#define GMM_FULL_GFXADDR_FROM_AUX_L3e_L2GFXADDR(L2GfxAddr) ((L2GfxAddr) << 16)
// Set the L2GfxAddr bit field given a full L2 graphics address
#define GMM_TO_AUX_L3e_L2GFXADDR(L2GfxAddress) ((L2GfxAddress) >> 16)
//===========================================================================
// typedef:
// GMM_AUXTTL2e
//
// Description:
// Struct for Auxiliary Translation-Table L2 entry
//--------------------------------------------------------------------------
typedef union GMM_AUXTTL2e_REC
{
struct
{
uint64_t Valid : 1;
uint64_t Reserved0 : 12;
uint64_t L1GfxAddr : 35;
uint64_t Reserved1 : 16;
};
uint64_t Value;
} GMM_AUXTTL2e;
C_ASSERT(sizeof(GMM_AUXTTL2e) == 8);
// Get the L1GfxAddr bit field as a full L1 graphics address
#define GMM_FULL_GFXADDR_FROM_AUX_L2e_L1GFXADDR(L1GfxAddr) ((L1GfxAddr) << 16)
// Set the L1GfxAddr bit field given a full L1 graphics address
#define GMM_TO_AUX_L2e_L1GFXADDR(L1GfxAddress) ((L1GfxAddress) >> 16)
typedef union GMM_AUXTTL1e_REC
{
struct
{
uint64_t Valid : 1;
uint64_t Mode : 2; //Compression ratio (128B compr ie 2:1 for RC, 256B compr ie 4:n compr for MC)
uint64_t Lossy : 1; //Lossy Compression
uint64_t Reserved0 : 2;
uint64_t Reserved2 : 2; //LSbs of 64B-aligned CCS chunk/cacheline address
uint64_t GfxAddress : 40; //256B-aligned CCS chunk address
uint64_t Reserved1 : 4;
uint64_t TileMode : 2; //Ys = 0, Y=1, Reserved=(2-3)
uint64_t Depth : 3; //Packed/Planar bit-depth for MC; Bpp for RC
uint64_t LumaChroma : 1; //Planar Y=0 or Cr=1
uint64_t Format : 6; //Format encoding shared with Vivante/Internal CC/DEC units to recognize surafce formats
};
uint64_t Value;
} GMM_AUXTTL1e;
C_ASSERT(sizeof(GMM_AUXTTL1e) == 8);
#define GMM_NO_TABLE ((GMM_GFX_ADDRESS)(-1L)) //common
#define GMM_INVALID_AUX_ENTRY ~__BIT(0)
#define GMM_AUX_L1e_SIZE (sizeof(GMM_AUXTTL1e))
#define GMM_AUX_L2e_SIZE (sizeof(GMM_AUXTTL2e))
#define GMM_AUX_L3e_SIZE (sizeof(GMM_AUXTTL3e))
#define GMM_AUX_L1_LOW_BIT (14)
#define GMM_AUX_L1_HIGH_BIT (23)
#define GMM_AUX_L2_LOW_BIT (24)
#define GMM_AUX_L2_HIGH_BIT (35)
#define GMM_AUX_L3_LOW_BIT (36)
#define GMM_AUX_L3_HIGH_BIT (47)
//For perf, AuxTable granularity changed to 64K
#define WA16K (pGmmGlobalContext->GetWaTable().WaAuxTable16KGranular)
// #L1 entries, i.e. 1024; 16K-granular ie 4 consequtive pages share Aux-cacheline;
// HW only tracks the distinct entries;
// Handle WA where HW chicken bit forces 64K-granularity
#define GMM_AUX_L1_SIZE(pGmmGlobalContext) ((1 << (GMM_AUX_L1_HIGH_BIT - GMM_AUX_L1_LOW_BIT + 1)) / (!(WA16K) ? 4 : 1))
#define GMM_AUX_L1_SIZE_DWORD(pGmmGlobalContext) (GFX_CEIL_DIV(GMM_AUX_L1_SIZE(pGmmGlobalContext), 32))
// #L2 entries, i.e. 4096
#define GMM_AUX_L2_SIZE (1 << (GMM_AUX_L2_HIGH_BIT - GMM_AUX_L2_LOW_BIT + 1))
#define GMM_AUX_L2_SIZE_DWORD (GFX_CEIL_DIV(GMM_AUX_L2_SIZE, 32))
// #L3 entries, i.e. 4096
#define GMM_AUX_L3_SIZE (1 << (GMM_AUX_L3_HIGH_BIT - GMM_AUX_L3_LOW_BIT + 1))
#define GMM_AUX_L1_ENTRY_IDX(GfxAddress,pGmmGlobalContext) \
((((GfxAddress) & GFX_MASK_LARGE(GMM_AUX_L1_LOW_BIT, GMM_AUX_L1_HIGH_BIT)) >> \
(uint64_t)GMM_AUX_L1_LOW_BIT) / (!(WA16K) ? 4 : 1))
#define GMM_AUX_L1_ENTRY_IDX_EXPORTED(GfxAddress,WA64KEx) \
((((GfxAddress) & GFX_MASK_LARGE(GMM_AUX_L1_LOW_BIT, GMM_AUX_L1_HIGH_BIT)) >> \
(uint64_t)GMM_AUX_L1_LOW_BIT) / ((WA64KEx) ? 4 : 1))
#define GMM_AUX_L2_ENTRY_IDX(GfxAddress) \
(((GfxAddress) & GFX_MASK_LARGE(GMM_AUX_L2_LOW_BIT, GMM_AUX_L2_HIGH_BIT)) >> \
(uint64_t)GMM_AUX_L2_LOW_BIT)
#define GMM_AUX_L3_ENTRY_IDX(GfxAddress) \
(((GfxAddress) & GFX_MASK_LARGE(GMM_AUX_L3_LOW_BIT, GMM_AUX_L3_HIGH_BIT)) >> \
(uint64_t)GMM_AUX_L3_LOW_BIT)
////////////////////// Auxiliary Translation Table definitions end//////////////////////////////////////////

165
Source/GmmLib/inc/External/Common/GmmPageTableMgr.h vendored Normal file
View File

@ -0,0 +1,165 @@
/*==============================================================================
Copyright(c) 2019 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files(the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and / or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
Description: This file contains the class definitions for GmmPageTableMgr
for user-mode PageTable management, that is common for both
Linux and Windows.
============================================================================*/
#pragma once
#include "GmmHw.h"
#ifdef __linux__
#include <pthread.h>
#endif
typedef enum _GMM_ENGINE_TYPE
{
ENGINE_TYPE_RCS = 0, //RCS
ENGINE_TYPE_COMPUTE = 1, //Compute-CS
ENGINE_TYPE_BCS, //BLT
ENGINE_TYPE_VD0,
ENGINE_TYPE_VD1,
ENGINE_TYPE_VE0
//Add all engines supporting AUX-TT
} GMM_ENGINE_TYPE;
typedef enum TT_Flags
{
AUXTT = 1, //Indicate TT request for AUX i.e. e2e compression
} TT_TYPE;
#if !(defined(__GMM_KMD__))
// Shared Structure for both Windows and Linux
typedef struct __GMM_DDI_UPDATEAUXTABLE
{
GMM_UMD_SYNCCONTEXT * UmdContext; // [in] pointer to thread-specific data, specifying BBQHandle/Fence etc
GMM_RESOURCE_INFO * BaseResInfo; // [in] GmmResourceInfo ptr for compressed resource
GMM_RESOURCE_INFO * AuxResInfo; // [in] GmmResourceInfo ptr for separate Auxiliary resource
GMM_GFX_ADDRESS BaseGpuVA; // [in] GPUVA where compressed resource has been mapped
GMM_GFX_ADDRESS AuxSurfVA; // [in] GPUVA where separate Auxiliary resource has been mapped
uint8_t Map; // [in] specifies if resource is being mapped or unmapped
uint8_t DoNotWait; // [in] specifies if PageTable update be done on CPU (true) or GPU (false)
}GMM_DDI_UPDATEAUXTABLE;
#ifdef __cplusplus
#include "GmmMemAllocator.hpp"
namespace GmmLib
{
class SyncInfoLin { //dummy class
public:
HANDLE BBQueueHandle;
uint64_t BBFence;
SyncInfoLin() {}
SyncInfoLin(HANDLE Handle, uint64_t Fence) {}
};
typedef class SyncInfoLin SyncInfo;
//Forward class declarations
class AuxTable;
class GmmPageTablePool;
typedef class GmmPageTablePool GMM_PAGETABLEPool;
typedef enum POOL_TYPE_REC
{
POOL_TYPE_TRTTL1 = 0,
POOL_TYPE_TRTTL2 = 1,
POOL_TYPE_AUXTTL1 = 2,
POOL_TYPE_AUXTTL2 = 3,
} POOL_TYPE;
//////////////////////////////////////////////////////////////////////////////////////////////
/// Contains functions and members for GMM_PAGETABLE_MGR, clients must place its pointer in
/// their device object. Clients call GmmLib to initialize the instance and use it for mapping
/// /unmapping on GmmLib managed page tables (TR-TT for SparseResources, AUX-TT for compression)
//////////////////////////////////////////////////////////////////////////////////////////////
class GMM_LIB_API NON_PAGED_SECTION GmmPageTableMgr :
public GmmMemAllocator
{
private:
GMM_ENGINE_TYPE EngType; //PageTable managed @ device-level (specifies engine associated with the device)
AuxTable* AuxTTObj; //Auxiliary Translation Table obj
GMM_PAGETABLEPool *pPool; //Common page table pool
uint32_t NumNodePoolElements;
GmmClientContext *pClientContext; ///< ClientContext of the client creating this Object
//OS-specific defn
#if defined __linux__
pthread_mutex_t PoolLock;
#endif
public:
GMM_DEVICE_CALLBACKS DeviceCb; //OS-specific defn: Will be used by Clients to send as input arguments for TR-TT APIs
GMM_DEVICE_CALLBACKS_INT DeviceCbInt; //OS-specific defn: Will be used internally GMM lib
GMM_TRANSLATIONTABLE_CALLBACKS TTCb; //OS-specific defn
HANDLE hCsr; // OCL per-device command stream receiver handle for aubcapture
public:
GmmPageTableMgr();
GmmPageTableMgr(GMM_DEVICE_CALLBACKS_INT *, uint32_t TTFlags, GmmClientContext *pClientContextIn); // Allocates memory for indicate TT’s root-tables, initializes common node-pool
//GMM_VIRTUAL GMM_GFX_ADDRESS GetTRL3TableAddr();
GMM_VIRTUAL GMM_GFX_ADDRESS GetAuxL3TableAddr();
//Update TT root table address in context-image
GMM_VIRTUAL GMM_STATUS InitContextAuxTableRegister(HANDLE initialBBHandle, GMM_ENGINE_TYPE engType); //Clients call it to update Aux-Table pointer in context-image, engType reqd. if @ context level
//Aux TT management API
GMM_VIRTUAL GMM_STATUS UpdateAuxTable(const GMM_DDI_UPDATEAUXTABLE*); //new API for updating Aux-Table to point to correct 16B-chunk
//for given host page VA when base/Aux surf is mapped/unmapped
GMM_VIRTUAL void __ReleaseUnusedPool(GMM_UMD_SYNCCONTEXT *UmdContext);
GMM_VIRTUAL GMM_PAGETABLEPool * __GetFreePoolNode(uint32_t * FreePoolNodeIdx, POOL_TYPE PoolType);
#if defined __linux__
//returns number of BOs for indicated TTs = NumNodePoolElements+1 BOs for root table and pools
GMM_VIRTUAL int GetNumOfPageTableBOs(uint8_t TTFlags);
//returns BO* list for indicated TT in client allocated memory
GMM_VIRTUAL int GetPageTableBOList(uint8_t TTFlags, void* BOList);
#endif
//Destructor
GMM_VIRTUAL ~GmmPageTableMgr(); //Clean-up page table structures
// Inline Functions
/////////////////////////////////////////////////////////////////////////////////////
/// Returns GmmClientContext associated with this PageTableMgr
/// @return ::GmmClientContext
/////////////////////////////////////////////////////////////////////////////////////
GMM_INLINE_VIRTUAL GmmClientContext* GetClientContext()
{
return pClientContext;
}
private:
GMM_PAGETABLEPool * __AllocateNodePool(uint32_t AddrAlignment, POOL_TYPE Type);
};
}
#endif // #ifdef __cplusplus
#endif

3
Source/GmmLib/inc/External/Common/GmmResourceInfo.h vendored
View File

@ -100,4 +100,5 @@ uint8_t __CanSupportStdTiling(GMM_TEXTURE_INFO Surface);
#ifdef __cplusplus
}
#endif /*__cplusplus*/
#endif /*__cplusplus*/

56
Source/GmmLib/inc/External/Common/GmmResourceInfoCommon.h vendored
View File

@ -90,6 +90,28 @@ namespace GmmLib
GMM_VIRTUAL bool ReAdjustPlaneProperties(bool IsAuxSurf);
GMM_VIRTUAL const GMM_PLATFORM_INFO& GetPlatformInfo();
/////////////////////////////////////////////////////////////////////////////////////
/// Returns tile mode for SURFACE_STATE programming.
/// @return Tiled Mode
/////////////////////////////////////////////////////////////////////////////////////
GMM_INLINE uint32_t GetTileModeSurfaceState(const GMM_TEXTURE_INFO *pTextureInfo) const
{
uint32_t TiledMode = 0;
if(GMM_IS_TILEY)
{
TiledMode =
pTextureInfo->Flags.Info.Linear ? 0 :
pTextureInfo->Flags.Info.TiledW ? 1 :
pTextureInfo->Flags.Info.TiledX ? 2 :
/* Y/YF/YS */ 3;
__GMM_ASSERT((TiledMode != 3) || (pTextureInfo->Flags.Info.TiledY || pTextureInfo->Flags.Info.TiledYf || pTextureInfo->Flags.Info.TiledYs));
}
return TiledMode;
}
public:
/* Constructors */
GmmResourceInfoCommon():
@ -138,6 +160,7 @@ namespace GmmLib
ClientType = rhs.ClientType;
Surf = rhs.Surf;
AuxSurf = rhs.AuxSurf;
AuxSecSurf = rhs.AuxSecSurf;
RotateInfo = rhs.RotateInfo;
ExistingSysMem = rhs.ExistingSysMem;
SvmAddress = rhs.SvmAddress;
@ -1292,12 +1315,15 @@ namespace GmmLib
}
else
{
switch (GetHAlign())
if(GMM_IS_TILEY)
{
case 4: HAlign = 1; break;
case 8: HAlign = 2; break;
case 16: HAlign = 3; break;
default: HAlign = 1;
switch (GetHAlign())
{
case 4: HAlign = 1; break;
case 8: HAlign = 2; break;
case 16: HAlign = 3; break;
default: HAlign = 1; // TODO(Benign): Change back to 0 + assert after packed YUV handling corrected.
}
}
}
}
@ -1362,16 +1388,16 @@ namespace GmmLib
/////////////////////////////////////////////////////////////////////////////////////
GMM_INLINE_VIRTUAL GMM_INLINE_EXPORTED uint32_t GMM_STDCALL GetTileModeSurfaceState()
{
uint32_t TiledMode = 0;
TiledMode =
Surf.Flags.Info.Linear ? 0 :
Surf.Flags.Info.TiledX ? 2 :
/* Y/YF/YS */ 3;
__GMM_ASSERT((TiledMode != 3) || (Surf.Flags.Info.TiledY || Surf.Flags.Info.TiledYf || Surf.Flags.Info.TiledYs));
return TiledMode;
return GetTileModeSurfaceState(&Surf);
}
/////////////////////////////////////////////////////////////////////////////////////
/// Returns tile mode for AUX SURFACE_STATE programming.
/// @return Tiled Mode
/////////////////////////////////////////////////////////////////////////////////////
GMM_INLINE_VIRTUAL GMM_INLINE_EXPORTED uint32_t GMM_STDCALL GetAuxTileModeSurfaceState()
{
return GetTileModeSurfaceState(&AuxSurf);
}
/////////////////////////////////////////////////////////////////////////////////////

4
Source/GmmLib/inc/External/Common/GmmResourceInfoExt.h vendored
View File

@ -181,7 +181,9 @@ typedef struct GMM_RESOURCE_ALIGNMENT_REC
// TBD - Place holder for GMM_RESOURCE_FLAG definition.
//---------------------------------------------------------------------------
#include "GmmResourceFlags.h"
#if defined __linux__
#include "External/Linux/GmmResourceInfoLinExt.h"
#endif
//==========================================================================
// typedef:

96
Source/GmmLib/inc/External/Linux/GmmResourceInfoLinExt.h vendored Normal file
View File

@ -0,0 +1,96 @@
/*==============================================================================
Copyright(c) 2019 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files(the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and / or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
============================================================================*/
#pragma once
#ifdef __linux__
#ifdef __cplusplus
extern "C" {
#endif /*__cplusplus*/
typedef struct {
int(*pfnAllocate)(void *bufMgr, size_t size, size_t alignment, void **bo, void **cpuAddr, uint64_t *gpuAddr);
void(*pfnDeallocate)(void *bo);
void(*pfnWaitFromCpu)(void *bo);;
} GMM_DEVICE_CB_PTRS;
typedef struct _GMM_DEVICE_CALLBACKS_INT
{
void *pBufMgr;
GMM_DEVICE_CB_PTRS DevCbPtrs_;
} GMM_DEVICE_CALLBACKS_INT;
// Add the definition to compatible.
typedef struct GMM_TRANSLATIONTABLE_CALLBACKS_REC
{
int (*pfPrologTranslationTable)(void *pDeviceHandle);
int (*pfWriteL1Entries)(void *pDeviceHandle,
const uint32_t NumEntries,
GMM_GFX_ADDRESS GfxAddress,
uint32_t *Data);
int (*pfWriteL2L3Entry)(void *pDeviceHandle,
GMM_GFX_ADDRESS GfxAddress,
uint64_t Data);
int (*pfWriteFenceID)(void *pDeviceHandle,
GMM_GFX_ADDRESS GfxAddress,
uint64_t Data);
int (*pfEpilogTranslationTable)(void *pDeviceHandle,
uint8_t ForceFlush);
int (*pfCopyL1Entry)(void *pDeviceHandle,
GMM_GFX_ADDRESS DstGfxAddress,
GMM_GFX_ADDRESS SrcGfxAddress);
int (*pfWriteL3Adr)(void *pDeviceHandle,
GMM_GFX_ADDRESS L3GfxAddress,
uint64_t RegOffset);
} GMM_TRANSLATIONTABLE_CALLBACKS;
typedef struct _GMM_DEVICE_CALLBACKS
{
void *pBufferMgr;
int FuncDevice;
int(*pfnAllocate)(void *bufMgr, size_t size, size_t alignment, void **bo, void **cpuAddr, uint64_t *gpuAddr);
void(*pfnDeallocate)(void *bo);
void(*pfnWaitFromCpu)(void *bo);;
} GMM_DEVICE_CALLBACKS;
// This definition is only for code sharing.
typedef struct GMM_UMD_SYNCCONTEXT_REC
{
void *pCommandQueueHandle; // pointer to command queue handle
void *pUpdateGpuVaInfo;
HANDLE BBFenceObj; // BatchBuffer Last Fence, for CPU to wait on before destroying TT pages
uint64_t BBLastFence; // BatchBuffer Last Fence for TT
} GMM_UMD_SYNCCONTEXT;
#ifdef __cplusplus
}
#endif /*__cplusplus*/
#endif /*__linux__*/

2
Source/GmmLib/inc/Internal/Common/GmmLibInc.h
View File

@ -29,6 +29,7 @@ OTHER DEALINGS IN THE SOFTWARE.
#include "External/Common/GmmConst.h"
#include "External/Common/GmmUtil.h"
#include "External/Common/GmmHw.h"
#include "External/Common/GmmCommonExt.h"
#include "External/Common/GmmPlatformExt.h"
@ -58,6 +59,7 @@ OTHER DEALINGS IN THE SOFTWARE.
#include "External/Common/GmmInfoExt.h"
#include "External/Common/GmmInfo.h"
#include "../Utility/GmmUtility.h"
#include "External/Common/GmmPageTableMgr.h"
#include "External/Common/GmmDebug.h" // Unified Definitions of GMM_ASSERT and GMM_DEBUG Macros

71
Source/GmmLib/inc/Internal/Linux/GmmResourceInfoLinInt.h Normal file
View File

@ -0,0 +1,71 @@
/*==============================================================================
Copyright(c) 2019 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files(the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and / or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
============================================================================*/
#pragma once
#if defined(__linux__) && !LHDM
// GMM Lib Client Exports
#include "External/Common/GmmCommonExt.h"
#include "External/Common/GmmUtil.h"
#include "External/Common/GmmResourceFlags.h"
#include "External/Common/GmmCachePolicy.h"
#include "External/Common/GmmCachePolicyExt.h"
#include "External/Common/GmmResourceInfoExt.h"
#include "External/Common/GmmPlatformExt.h"
#include "External/Common/GmmTextureExt.h"
#include "External/Common/GmmInfoExt.h"
#include "External/Common/GmmResourceInfo.h"
#include "External/Common/GmmInfo.h"
typedef struct {
size_t size;
size_t alignment;
void *bo;
void *cpuAddr;
uint64_t gfxAddr;
} GMM_DDI_ALLOCATE;
typedef struct {
void *bo;
} GMM_DDI_DEALLOCATE;
typedef struct {
void *bo;
}GMM_DDI_WAITFORSYNCHRONIZATIONOBJECTFROMCPU;
typedef enum GMM_DEVICE_CALLBACKS_TYPE_REC
{
GMM_DEV_CB_ALLOC = 0,
GMM_DEV_CB_DEALLOC,
GMM_DEV_CB_WAIT_FROM_CPU,
} GMM_DEVICE_CALLBACKS_TYPE;
int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_ALLOCATE *pAllocate);
int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_DEALLOCATE *pDeallocate);
int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_WAITFORSYNCHRONIZATIONOBJECTFROMCPU *pWait);
int GmmCheckForNullDevCbPfn(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DEVICE_CALLBACKS_TYPE CallBackType);
extern GMM_TRANSLATIONTABLE_CALLBACKS DummyTTCB;
#endif /*__linux__*/

18
Source/inc/common/sku_wa.h
View File

@ -457,6 +457,24 @@ typedef struct _WA_TABLE
WA_BUG_TYPE_CORRUPTION,
WA_BUG_PERF_IMPACT_UNKNOWN, WA_COMPONENT_GMM)
WA_DECLARE(
WaAuxTable16KGranular,
"AuxTable map granularity changed to 16K ",
WA_BUG_TYPE_PERF,
WA_BUG_PERF_IMPACT_UNKNOWN, WA_COMPONENT_UNKNOWN)
WA_DECLARE(
WaLimit128BMediaCompr,
"WA to limit media decompression on Render pipe to 128B (2CLs) 4:n.",
WA_BUG_TYPE_FUNCTIONAL,
WA_BUG_PERF_IMPACT_UNKNOWN, WA_COMPONENT_GMM)
WA_DECLARE(
WaUntypedBufferCompression,
"WA to allow untyped raw buffer AuxTable mapping",
WA_BUG_TYPE_FUNCTIONAL,
WA_BUG_PERF_IMPACT_UNKNOWN, WA_COMPONENT_GMM)
} WA_TABLE, *PWA_TABLE;
//********************************** SKU/WA Macros *************************************