Prepare to use gmm as dll on Windows

Since this commit neo on Windows can use static or shared gmm lib

Change-Id: I7db70d7f9bc969e8193ac77e8b6d65ecc57d0093
This commit is contained in:
Mateusz Jablonski
2018-05-17 11:47:35 +02:00
committed by sys_ocldev
parent c1782b802a
commit 98b8b4b6a4
34 changed files with 1022 additions and 87 deletions

View File

@ -100,7 +100,11 @@ target_compile_definitions(${NEO_STATIC_LIB_NAME} PUBLIC DEFAULT_PLATFORM=${DEFA
link_directories(${GMM_LIB_PATHS})
target_link_libraries(${NEO_STATIC_LIB_NAME} ${GMMUMD_LIB_NAME})
if(UNIX OR USE_STATIC_GMM)
target_link_libraries(${NEO_STATIC_LIB_NAME} ${GMMUMD_LIB_NAME})
else()
target_compile_definitions(${NEO_STATIC_LIB_NAME} PUBLIC GMM_LIB_DLL)
endif()
if(INSTRUMENTATION_LIB_NAME)
add_dependencies(${NEO_STATIC_LIB_NAME} ${INSTRUMENTATION_LIB_NAME})
@ -134,6 +138,7 @@ if(${GENERATE_EXECUTABLE})
add_library(${NEO_DYNAMIC_LIB_NAME} SHARED
${NEO_DYNAMIC_LIB__TARGET_OBJECTS}
)
add_dependencies(${NEO_DYNAMIC_LIB_NAME} ${GMMUMD_LIB_NAME})
if(GTPIN_HEADERS_DIR)
macro(macro_for_each_gen)

View File

@ -19,6 +19,7 @@
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "lib_names.h"
namespace Os {
#if defined(_WIN64)
@ -30,4 +31,5 @@ const char *igcDllName = "igc32.dll";
#endif
const char *gdiDllName = "gdi32.dll";
const char *gmmDllName = GMM_LIB_FILENAME;
} // namespace Os

View File

@ -31,28 +31,6 @@
#include "runtime/helpers/hw_info.h"
#include "runtime/sku_info/operations/sku_info_transfer.h"
extern "C" {
void GMMDebugBreak(const char *file, const char *function, const int line) {
}
void GMMPrintMessage(uint32_t debugLevel, const char *debugMessageFmt, ...) {
}
typedef struct GfxDebugControlRec {
uint32_t Version;
uint32_t Size;
uint32_t AssertEnableMask;
uint32_t EnableDebugFileDump;
uint32_t DebugEnableMask;
uint32_t RingBufDbgMask;
uint32_t ReportAssertEnable;
uint32_t AssertBreakDisable;
} GFX_DEBUG_CONTROL, *PGFX_DEBUG_CONTROL;
PGFX_DEBUG_CONTROL pDebugControl;
}
namespace OCLRT {
void Gmm::create() {
if (resourceParams.BaseWidth >= maxPossiblePitch) {
@ -71,19 +49,21 @@ bool Gmm::initContext(const PLATFORM *pPlatform,
_WA_TABLE gmmWaTable = {};
SkuInfoTransfer::transferFtrTableForGmm(&gmmFtrTable, pSkuTable);
SkuInfoTransfer::transferWaTableForGmm(&gmmWaTable, pWaTable);
bool success = GMM_SUCCESS == GmmInitGlobalContext(*pPlatform, &gmmFtrTable, &gmmWaTable, pGtSysInfo, GMM_CLIENT::GMM_OCL_VISTA);
if (!isLoaded) {
loadLib();
}
bool success = GMM_SUCCESS == initGlobalContextFunc(*pPlatform, &gmmFtrTable, &gmmWaTable, pGtSysInfo, GMM_CLIENT::GMM_OCL_VISTA);
UNRECOVERABLE_IF(!success);
Gmm::gmmClientContext = GmmCreateClientContext(GMM_CLIENT::GMM_OCL_VISTA);
Gmm::gmmClientContext = createClientContextFunc(GMM_CLIENT::GMM_OCL_VISTA);
}
return Gmm::gmmClientContext != nullptr;
}
void Gmm::destroyContext() {
if (Gmm::gmmClientContext) {
GmmDeleteClientContext(Gmm::gmmClientContext);
deleteClientContextFunc(Gmm::gmmClientContext);
Gmm::gmmClientContext = nullptr;
GmmDestroyGlobalContext();
destroyGlobalContextFunc();
}
}
@ -414,5 +394,6 @@ bool Gmm::unifiedAuxTranslationCapable() const {
bool Gmm::useSimplifiedMocsTable = false;
GMM_CLIENT_CONTEXT *Gmm::gmmClientContext = nullptr;
bool Gmm::isLoaded = false;
} // namespace OCLRT

View File

@ -27,12 +27,6 @@
#include "runtime/gmm_helper/gmm_lib.h"
#include "runtime/api/cl_types.h"
extern "C" {
void GMMDebugBreak(const char *file, const char *function, const int line);
void GMMPrintMessage(uint32_t debugLevel, const char *debugMessageFmt, ...);
}
namespace OCLRT {
struct HardwareInfo;
struct FeatureTable;
@ -63,6 +57,7 @@ class Gmm {
static Gmm *create(GMM_RESOURCE_INFO *inputGmm);
static bool initContext(const PLATFORM *pPlatform, const FeatureTable *pSkuTable, const WorkaroundTable *pWaTable, const GT_SYSTEM_INFO *pGtSysInfo);
static void loadLib();
static void destroyContext();
static uint32_t getMOCS(uint32_t type);
@ -97,8 +92,14 @@ class Gmm {
GMM_RESCREATE_PARAMS resourceParams = {};
std::unique_ptr<GmmResourceInfo> gmmResourceInfo;
static decltype(&GmmInitGlobalContext) initGlobalContextFunc;
static decltype(&GmmDestroyGlobalContext) destroyGlobalContextFunc;
static decltype(&GmmCreateClientContext) createClientContextFunc;
static decltype(&GmmDeleteClientContext) deleteClientContextFunc;
bool isRenderCompressed = false;
static bool useSimplifiedMocsTable;
static GMM_CLIENT_CONTEXT *gmmClientContext;
static bool isLoaded;
};
} // namespace OCLRT

View File

@ -46,7 +46,7 @@ bool GmmMemoryBase::configureDeviceAddressSpace(GMM_ESCAPE_HANDLE hAdapter,
}
uintptr_t GmmMemoryBase::getInternalGpuVaRangeLimit() {
return static_cast<uintptr_t>(pGmmGlobalContext->GetInternalGpuVaRangeLimit());
return static_cast<uintptr_t>(Gmm::gmmClientContext->GetInternalGpuVaRangeLimit());
}
}; // namespace OCLRT

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, Intel Corporation
* Copyright (c) 2017 - 2018, Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@ -24,7 +24,7 @@
#include "runtime/gmm_helper/page_table_mngr.h"
namespace OCLRT {
GmmPageTableMngr *GmmPageTableMngr::create(GMM_DEVICE_CALLBACKS *deviceCb, unsigned int translationTableFlags, GMM_TRANSLATIONTABLE_CALLBACKS *translationTableCb) {
GmmPageTableMngr *GmmPageTableMngr::create(GMM_DEVICE_CALLBACKS_INT *deviceCb, unsigned int translationTableFlags, GMM_TRANSLATIONTABLE_CALLBACKS *translationTableCb) {
return new GmmPageTableMngr(deviceCb, translationTableFlags, translationTableCb);
}

View File

@ -30,7 +30,7 @@ class GmmPageTableMngr {
public:
MOCKABLE_VIRTUAL ~GmmPageTableMngr() = default;
static GmmPageTableMngr *create(GMM_DEVICE_CALLBACKS *deviceCb, unsigned int translationTableFlags, GMM_TRANSLATIONTABLE_CALLBACKS *translationTableCb);
static GmmPageTableMngr *create(GMM_DEVICE_CALLBACKS_INT *deviceCb, unsigned int translationTableFlags, GMM_TRANSLATIONTABLE_CALLBACKS *translationTableCb);
MOCKABLE_VIRTUAL GMM_STATUS initContextAuxTableRegister(HANDLE initialBBHandle, GMM_ENGINE_TYPE engineType) {
return pageTableManager->InitContextAuxTableRegister(initialBBHandle, engineType);
@ -50,7 +50,7 @@ class GmmPageTableMngr {
GmmPageTableMngr() = default;
GmmPageTableMngr(GMM_DEVICE_CALLBACKS *deviceCb, unsigned int translationTableFlags, GMM_TRANSLATIONTABLE_CALLBACKS *translationTableCb);
GmmPageTableMngr(GMM_DEVICE_CALLBACKS_INT *deviceCb, unsigned int translationTableFlags, GMM_TRANSLATIONTABLE_CALLBACKS *translationTableCb);
UniquePtrType pageTableManager;
};

View File

@ -28,7 +28,7 @@ void GmmPageTableMngr::customDeleter(GMM_PAGETABLE_MGR *gmmPageTableManager) {
Gmm::gmmClientContext->DestroyPageTblMgrObject(gmmPageTableManager);
}
GmmPageTableMngr::GmmPageTableMngr(GMM_DEVICE_CALLBACKS *deviceCb, unsigned int translationTableFlags, GMM_TRANSLATIONTABLE_CALLBACKS *translationTableCb) {
GmmPageTableMngr::GmmPageTableMngr(GMM_DEVICE_CALLBACKS_INT *deviceCb, unsigned int translationTableFlags, GMM_TRANSLATIONTABLE_CALLBACKS *translationTableCb) {
auto pageTableMngrPtr = Gmm::gmmClientContext->CreatePageTblMgrObject(deviceCb, translationTableCb, translationTableFlags);
this->pageTableManager = UniquePtrType(pageTableMngrPtr, GmmPageTableMngr::customDeleter);
}

View File

@ -43,6 +43,7 @@ set(RUNTIME_SRCS_OS_INTERFACE_LINUX
${CMAKE_CURRENT_SOURCE_DIR}/drm_neo.h
${CMAKE_CURRENT_SOURCE_DIR}/drm_neo_create.cpp
${CMAKE_CURRENT_SOURCE_DIR}/drm_null_device.h
${CMAKE_CURRENT_SOURCE_DIR}/gmm_interface_linux.cpp
${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config.cpp
${CMAKE_CURRENT_SOURCE_DIR}/linux_inc.cpp
${CMAKE_CURRENT_SOURCE_DIR}/os_thread_linux.cpp

View File

@ -0,0 +1,39 @@
/*
* Copyright (c) 2018, Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "runtime/gmm_helper/gmm_helper.h"
namespace OCLRT {
decltype(Gmm::initGlobalContextFunc) Gmm::initGlobalContextFunc = nullptr;
decltype(Gmm::destroyGlobalContextFunc) Gmm::destroyGlobalContextFunc = nullptr;
decltype(Gmm::createClientContextFunc) Gmm::createClientContextFunc = nullptr;
decltype(Gmm::deleteClientContextFunc) Gmm::deleteClientContextFunc = nullptr;
void Gmm::loadLib() {
Gmm::initGlobalContextFunc = GmmInitGlobalContext;
Gmm::destroyGlobalContextFunc = GmmDestroyGlobalContext;
Gmm::createClientContextFunc = GmmCreateClientContext;
Gmm::deleteClientContextFunc = GmmDeleteClientContext;
isLoaded = true;
}
}

View File

@ -24,6 +24,13 @@ else()
set(KMDAF_FILE_SUFFIX "_stub")
endif()
if(USE_STATIC_GMM)
set(GMM_INTERFACE_FILE_WINDOWS ${CMAKE_CURRENT_SOURCE_DIR}/gmm_interface_static_win.cpp)
else()
set(GMM_INTERFACE_FILE_WINDOWS ${CMAKE_CURRENT_SOURCE_DIR}/gmm_interface_dynamic_win.cpp)
endif()
set_property(GLOBAL PROPERTY GMM_INTERFACE_FILE_WINDOWS ${GMM_INTERFACE_FILE_WINDOWS})
set(RUNTIME_SRCS_OS_INTERFACE_WINDOWS
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
${CMAKE_CURRENT_SOURCE_DIR}/api_win.cpp
@ -39,6 +46,7 @@ set(RUNTIME_SRCS_OS_INTERFACE_WINDOWS
${CMAKE_CURRENT_SOURCE_DIR}/driver_info.h
${CMAKE_CURRENT_SOURCE_DIR}/gdi_interface.cpp
${CMAKE_CURRENT_SOURCE_DIR}/gdi_interface.h
${GMM_INTERFACE_FILE_WINDOWS}
${CMAKE_CURRENT_SOURCE_DIR}/kmdaf_listener${KMDAF_FILE_SUFFIX}.cpp
${CMAKE_CURRENT_SOURCE_DIR}/kmdaf_listener.h
${CMAKE_CURRENT_SOURCE_DIR}/os_inc.h

View File

@ -99,7 +99,5 @@ class Gdi {
private:
OCLRT::Windows::OsLibrary gdiDll;
static const std::string gdiDllName;
static const std::string gdiMockDllName;
};
} // namespace OCLRT

View File

@ -0,0 +1,61 @@
/*
* Copyright (c) 2018, Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "runtime/gmm_helper/gmm_helper.h"
#include "runtime/helpers/debug_helpers.h"
#include "runtime/os_interface/os_library.h"
namespace Os {
extern const char *gmmDllName;
}
namespace OCLRT {
GMM_STATUS(GMM_STDCALL *myPfnCreateSingletonContext)
(const PLATFORM Platform, const SKU_FEATURE_TABLE *pSkuTable, const WA_TABLE *pWaTable, const GT_SYSTEM_INFO *pGtSysInfo);
GMM_STATUS GMM_STDCALL myGmmInitGlobalContext(const PLATFORM Platform, const SKU_FEATURE_TABLE *pSkuTable, const WA_TABLE *pWaTable, const GT_SYSTEM_INFO *pGtSysInfo, GMM_CLIENT ClientType) {
return myPfnCreateSingletonContext(Platform, pSkuTable, pWaTable, pGtSysInfo);
}
decltype(Gmm::initGlobalContextFunc) Gmm::initGlobalContextFunc = &myGmmInitGlobalContext;
decltype(Gmm::destroyGlobalContextFunc) Gmm::destroyGlobalContextFunc = nullptr;
decltype(Gmm::createClientContextFunc) Gmm::createClientContextFunc = nullptr;
decltype(Gmm::deleteClientContextFunc) Gmm::deleteClientContextFunc = nullptr;
std::unique_ptr<OsLibrary> gmmLib;
void Gmm::loadLib() {
gmmLib.reset(OsLibrary::load(Os::gmmDllName));
UNRECOVERABLE_IF(!gmmLib);
if (gmmLib->isLoaded()) {
auto openGmmFunc = reinterpret_cast<decltype(&OpenGmm)>(gmmLib->getProcAddress(GMM_ENTRY_NAME));
GmmExportEntries entries;
auto status = openGmmFunc(&entries);
if (status == GMM_SUCCESS) {
myPfnCreateSingletonContext = entries.pfnCreateSingletonContext;
Gmm::destroyGlobalContextFunc = entries.pfnDestroySingletonContext;
Gmm::createClientContextFunc = entries.pfnCreateClientContext;
Gmm::deleteClientContextFunc = entries.pfnDeleteClientContext;
isLoaded = myPfnCreateSingletonContext && Gmm::destroyGlobalContextFunc && Gmm::createClientContextFunc && Gmm::deleteClientContextFunc;
}
}
UNRECOVERABLE_IF(!isLoaded);
}
}

View File

@ -0,0 +1,58 @@
/*
* Copyright (c) 2018, Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "runtime/gmm_helper/gmm_helper.h"
namespace OCLRT {
decltype(Gmm::initGlobalContextFunc) Gmm::initGlobalContextFunc = nullptr;
decltype(Gmm::destroyGlobalContextFunc) Gmm::destroyGlobalContextFunc = nullptr;
decltype(Gmm::createClientContextFunc) Gmm::createClientContextFunc = nullptr;
decltype(Gmm::deleteClientContextFunc) Gmm::deleteClientContextFunc = nullptr;
void Gmm::loadLib() {
Gmm::initGlobalContextFunc = GmmInitGlobalContext;
Gmm::destroyGlobalContextFunc = GmmDestroyGlobalContext;
Gmm::createClientContextFunc = GmmCreateClientContext;
Gmm::deleteClientContextFunc = GmmDeleteClientContext;
isLoaded = true;
}
}
extern "C" {
void GMMDebugBreak(const char *file, const char *function, const int line) {
}
void GMMPrintMessage(uint32_t debugLevel, const char *debugMessageFmt, ...) {
}
typedef struct GfxDebugControlRec {
uint32_t Version;
uint32_t Size;
uint32_t AssertEnableMask;
uint32_t EnableDebugFileDump;
uint32_t DebugEnableMask;
uint32_t RingBufDbgMask;
uint32_t ReportAssertEnable;
uint32_t AssertBreakDisable;
} GFX_DEBUG_CONTROL, *PGFX_DEBUG_CONTROL;
PGFX_DEBUG_CONTROL pDebugControl;
}

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, Intel Corporation
* Copyright (c) 2017 - 2018, Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),

View File

@ -176,27 +176,27 @@ bool WddmCommandStreamReceiver<GfxFamily>::waitForFlushStamp(FlushStamp &flushSt
template <typename GfxFamily>
GmmPageTableMngr *WddmCommandStreamReceiver<GfxFamily>::createPageTableManager() {
GMM_DEVICE_CALLBACKS deviceCallbacks = {};
GMM_DEVICE_CALLBACKS_INT deviceCallbacks = {};
GMM_TRANSLATIONTABLE_CALLBACKS ttCallbacks = {};
auto gdi = wddm->getGdi();
// clang-format off
deviceCallbacks.Adapter = wddm->getAdapter();
deviceCallbacks.hDevice = wddm->getDevice();
deviceCallbacks.Adapter.KmtHandle = wddm->getAdapter();
deviceCallbacks.hDevice.KmtHandle = wddm->getDevice();
deviceCallbacks.PagingQueue = wddm->getPagingQueue();
deviceCallbacks.PagingFence = wddm->getPagingQueueSyncObject();
deviceCallbacks.pfnAllocate = gdi->createAllocation;
deviceCallbacks.pfnDeallocate = gdi->destroyAllocation;
deviceCallbacks.pfnMapGPUVA = gdi->mapGpuVirtualAddress;
deviceCallbacks.pfnMakeResident = gdi->makeResident;
deviceCallbacks.pfnEvict = gdi->evict;
deviceCallbacks.pfnReserveGPUVA = gdi->reserveGpuVirtualAddress;
deviceCallbacks.pfnUpdateGPUVA = gdi->updateGpuVirtualAddress;
deviceCallbacks.pfnWaitFromCpu = gdi->waitForSynchronizationObjectFromCpu;
deviceCallbacks.pfnLock = gdi->lock2;
deviceCallbacks.pfnUnLock = gdi->unlock2;
deviceCallbacks.pfnEscape = gdi->escape;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnAllocate = gdi->createAllocation;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnDeallocate = gdi->destroyAllocation;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnMapGPUVA = gdi->mapGpuVirtualAddress;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnMakeResident = gdi->makeResident;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnEvict = gdi->evict;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnReserveGPUVA = gdi->reserveGpuVirtualAddress;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnUpdateGPUVA = gdi->updateGpuVirtualAddress;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnWaitFromCpu = gdi->waitForSynchronizationObjectFromCpu;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnLock = gdi->lock2;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnUnLock = gdi->unlock2;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnEscape = gdi->escape;
ttCallbacks.pfWriteL3Adr = TTCallbacks<GfxFamily>::writeL3Address;
// clang-format on