Enable L1 cache for Tigerlake

Change-Id: I33513ed084f9d06ceca11315cac03f1b682db535
Signed-off-by: Maciej Dziuban <maciej.dziuban@intel.com>
Related-To: NEO-4832

opencl/source/mem_obj/buffer_base.inl

@@ -36,10 +36,11 @@ template <typename GfxFamily>
 void BufferHw<GfxFamily>::setArgStateful(void *memory, bool forceNonAuxMode, bool disableL3, bool alignSizeForAuxTranslation, bool isReadOnlyArgument, const Device &device) {
     auto rootDeviceIndex = device.getRootDeviceIndex();
     auto graphicsAllocation = multiGraphicsAllocation.getGraphicsAllocation(rootDeviceIndex);
+    const auto isReadOnly = isValueSet(getFlags(), CL_MEM_READ_ONLY) || isReadOnlyArgument;
     EncodeSurfaceState<GfxFamily>::encodeBuffer(memory, getBufferAddress(rootDeviceIndex),
                                                 getSurfaceSize(alignSizeForAuxTranslation, rootDeviceIndex),
-                                                getMocsValue(disableL3, isReadOnlyArgument, rootDeviceIndex),
-                                                true, forceNonAuxMode, device.getNumAvailableDevices(),
+                                                getMocsValue(disableL3, isReadOnly, rootDeviceIndex),
+                                                true, forceNonAuxMode, isReadOnly, device.getNumAvailableDevices(),
                                                 graphicsAllocation, device.getGmmHelper());
     appendSurfaceStateExt(memory);
 }

opencl/test/unit_test/gen12lp/CMakeLists.txt

@@ -8,6 +8,7 @@ if(TESTS_GEN12LP)
   set(IGDRCL_SRCS_tests_gen12lp
       ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
       ${CMAKE_CURRENT_SOURCE_DIR}/aub_command_stream_receiver_tests_gen12lp.inl
+      ${CMAKE_CURRENT_SOURCE_DIR}/buffer_tests_gen12lp.inl
       ${CMAKE_CURRENT_SOURCE_DIR}/coherency_tests_gen12lp.inl
       ${CMAKE_CURRENT_SOURCE_DIR}/command_stream_receiver_hw_tests_gen12lp.inl
       ${CMAKE_CURRENT_SOURCE_DIR}/command_stream_receiver_simulated_common_hw_tests_gen12lp.inl

opencl/test/unit_test/gen12lp/buffer_tests_gen12lp.inl

@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2019-2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/device/device.h"
+#include "shared/source/gmm_helper/gmm_helper.h"
+#include "shared/test/unit_test/helpers/debug_manager_state_restore.h"
+
+#include "opencl/source/cl_device/cl_device.h"
+#include "opencl/source/mem_obj/buffer.h"
+#include "opencl/test/unit_test/mocks/mock_context.h"
+#include "test.h"
+
+using namespace NEO;
+
+struct BufferTestsTgllp : ::testing::Test {
+    void SetUp() override {
+        context = std::make_unique<MockContext>();
+        device = context->getDevice(0);
+    }
+
+    std::unique_ptr<MockContext> context{};
+    ClDevice *device{};
+
+    cl_int retVal = CL_SUCCESS;
+};
+
+GEN12LPTEST_F(BufferTestsTgllp, givenBufferNotReadonlyWhenProgrammingSurfaceStateThenUseL3) {
+    auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+        context.get(),
+        CL_MEM_READ_WRITE,
+        MemoryConstants::pageSize,
+        nullptr,
+        retVal));
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    typename FamilyType::RENDER_SURFACE_STATE surfaceState = {};
+    buffer->setArgStateful(&surfaceState, false, false, false, false, device->getDevice());
+
+    const auto expectedMocs = device->getGmmHelper()->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER);
+    const auto actualMocs = surfaceState.getMemoryObjectControlState();
+    EXPECT_EQ(expectedMocs, actualMocs);
+}
+
+GEN12LPTEST_F(BufferTestsTgllp, givenBufferReadonlyWhenProgrammingSurfaceStateThenUseL1) {
+    auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+        context.get(),
+        CL_MEM_READ_WRITE,
+        MemoryConstants::pageSize,
+        nullptr,
+        retVal));
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    typename FamilyType::RENDER_SURFACE_STATE surfaceState = {};
+    buffer->setArgStateful(&surfaceState, false, false, false, true, context->getDevice(0)->getDevice());
+
+    const auto expectedMocs = device->getGmmHelper()->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CONST);
+    const auto actualMocs = surfaceState.getMemoryObjectControlState();
+    EXPECT_EQ(expectedMocs, actualMocs);
+}
+
+GEN12LPTEST_F(BufferTestsTgllp, givenConstantSurfaceWhenProgrammingSurfaceStateThenUseL1) {
+    auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+        context.get(),
+        CL_MEM_READ_WRITE,
+        MemoryConstants::pageSize,
+        nullptr,
+        retVal));
+    ASSERT_EQ(CL_SUCCESS, retVal);
+    buffer->getGraphicsAllocation(0)->setAllocationType(GraphicsAllocation::AllocationType::CONSTANT_SURFACE);
+
+    typename FamilyType::RENDER_SURFACE_STATE surfaceState = {};
+    buffer->setArgStateful(&surfaceState, false, false, false, false, context->getDevice(0)->getDevice());
+
+    const auto expectedMocs = device->getGmmHelper()->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CONST);
+    const auto actualMocs = surfaceState.getMemoryObjectControlState();
+    EXPECT_EQ(expectedMocs, actualMocs);
+}
+
+GEN12LPTEST_F(BufferTestsTgllp, givenL1ForceEnabledWhenProgrammingSurfaceStateThenUseL1) {
+    DebugManagerStateRestore restore{};
+    DebugManager.flags.ForceL1Caching.set(1);
+
+    auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+        context.get(),
+        CL_MEM_READ_WRITE,
+        MemoryConstants::pageSize,
+        nullptr,
+        retVal));
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    typename FamilyType::RENDER_SURFACE_STATE surfaceState = {};
+    buffer->setArgStateful(&surfaceState, false, false, false, false, device->getDevice());
+
+    const auto expectedMocs = device->getGmmHelper()->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CONST);
+    const auto actualMocs = surfaceState.getMemoryObjectControlState();
+    EXPECT_EQ(expectedMocs, actualMocs);
+}
+
+GEN12LPTEST_F(BufferTestsTgllp, givenBufferReadonlyL1ForceDisabledWhenProgrammingSurfaceStateThenUseL3) {
+    DebugManagerStateRestore restore{};
+    DebugManager.flags.ForceL1Caching.set(0);
+
+    auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+        context.get(),
+        CL_MEM_READ_WRITE,
+        MemoryConstants::pageSize,
+        nullptr,
+        retVal));
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    typename FamilyType::RENDER_SURFACE_STATE surfaceState = {};
+    buffer->setArgStateful(&surfaceState, false, false, false, true, device->getDevice());
+
+    const auto expectedMocs = device->getGmmHelper()->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER);
+    const auto actualMocs = surfaceState.getMemoryObjectControlState();
+    EXPECT_EQ(expectedMocs, actualMocs);
+}

opencl/test/unit_test/gen12lp/dg1/excludes_dg1.cpp

@@ -26,3 +26,7 @@ HWCMDTEST_EXCLUDE_FAMILY(TimestampEventCreate, givenEventTimestampsWhenQueryKern
 HWCMDTEST_EXCLUDE_FAMILY(ProfilingCommandsTest, givenKernelWhenProfilingCommandStartIsTakenThenTimeStampAddressIsProgrammedCorrectly, IGFX_DG1);
 
 HWCMDTEST_EXCLUDE_FAMILY(ProfilingCommandsTest, givenKernelWhenProfilingCommandStartIsNotTakenThenTimeStampAddressIsProgrammedCorrectly, IGFX_DG1);
+
+HWCMDTEST_EXCLUDE_FAMILY(BufferSetSurfaceTests, givenBufferSetSurfaceThatMemoryIsUnalignedToCachelineButReadOnlyThenL3CacheShouldBeStillOn, IGFX_DG1)
+
+HWCMDTEST_EXCLUDE_FAMILY(BufferSetSurfaceTests, givenAlignedCacheableReadOnlyBufferThenChoseOclBufferPolicy, IGFX_DG1);

opencl/test/unit_test/gen12lp/gen12lp_tests_wrapper.cpp

@@ -6,6 +6,7 @@
  */
 
 #include "opencl/test/unit_test/gen12lp/aub_command_stream_receiver_tests_gen12lp.inl"
+#include "opencl/test/unit_test/gen12lp/buffer_tests_gen12lp.inl"
 #include "opencl/test/unit_test/gen12lp/coherency_tests_gen12lp.inl"
 #include "opencl/test/unit_test/gen12lp/command_stream_receiver_hw_tests_gen12lp.inl"
 #include "opencl/test/unit_test/gen12lp/command_stream_receiver_simulated_common_hw_tests_gen12lp.inl"

opencl/test/unit_test/gen12lp/hw_helper_tests_gen12lp.inl

@@ -378,3 +378,49 @@ GEN12LPTEST_F(HwHelperTestGen12Lp, givenGen12WhenCallIsPackedSupportedThenReturn
     auto &helper = HwHelper::get(renderCoreFamily);
     EXPECT_TRUE(helper.packedFormatsSupported());
 }
+
+GEN12LPTEST_F(HwHelperTestGen12Lp, whenRequestingMocsThenProperMocsIndicesAreBeingReturned) {
+    auto &helper = HwHelper::get(renderCoreFamily);
+    auto gmmHelper = this->pDevice->getGmmHelper();
+
+    const auto mocsNoCache = gmmHelper->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CACHELINE_MISALIGNED) >> 1;
+    const auto mocsL3 = gmmHelper->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER) >> 1;
+    const auto mocsL1 = gmmHelper->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CONST) >> 1;
+
+    EXPECT_EQ(mocsNoCache, helper.getMocsIndex(*gmmHelper, false, false));
+    EXPECT_EQ(mocsNoCache, helper.getMocsIndex(*gmmHelper, false, true));
+    EXPECT_EQ(mocsL3, helper.getMocsIndex(*gmmHelper, true, false));
+    EXPECT_EQ(mocsL1, helper.getMocsIndex(*gmmHelper, true, true));
+}
+
+GEN12LPTEST_F(HwHelperTestGen12Lp, givenL1ForceEnabledWhenRequestingMocsThenProperMocsIndicesAreBeingReturned) {
+    DebugManagerStateRestore restore{};
+    DebugManager.flags.ForceL1Caching.set(1);
+
+    auto &helper = HwHelper::get(renderCoreFamily);
+    auto gmmHelper = this->pDevice->getGmmHelper();
+
+    const auto mocsNoCache = gmmHelper->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CACHELINE_MISALIGNED) >> 1;
+    const auto mocsL1 = gmmHelper->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CONST) >> 1;
+
+    EXPECT_EQ(mocsNoCache, helper.getMocsIndex(*gmmHelper, false, false));
+    EXPECT_EQ(mocsNoCache, helper.getMocsIndex(*gmmHelper, false, true));
+    EXPECT_EQ(mocsL1, helper.getMocsIndex(*gmmHelper, true, false));
+    EXPECT_EQ(mocsL1, helper.getMocsIndex(*gmmHelper, true, true));
+}
+
+GEN12LPTEST_F(HwHelperTestGen12Lp, givenL1ForceDisabledWhenRequestingMocsThenProperMocsIndicesAreBeingReturned) {
+    DebugManagerStateRestore restore{};
+    DebugManager.flags.ForceL1Caching.set(0);
+
+    auto &helper = HwHelper::get(renderCoreFamily);
+    auto gmmHelper = this->pDevice->getGmmHelper();
+
+    const auto mocsNoCache = gmmHelper->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CACHELINE_MISALIGNED) >> 1;
+    const auto mocsL3 = gmmHelper->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER) >> 1;
+
+    EXPECT_EQ(mocsNoCache, helper.getMocsIndex(*gmmHelper, false, false));
+    EXPECT_EQ(mocsNoCache, helper.getMocsIndex(*gmmHelper, false, true));
+    EXPECT_EQ(mocsL3, helper.getMocsIndex(*gmmHelper, true, false));
+    EXPECT_EQ(mocsL3, helper.getMocsIndex(*gmmHelper, true, true));
+}

opencl/test/unit_test/gen12lp/rkl/excludes_rkl.cpp

@@ -26,3 +26,7 @@ HWCMDTEST_EXCLUDE_FAMILY(TimestampEventCreate, givenEventTimestampsWhenQueryKern
 HWCMDTEST_EXCLUDE_FAMILY(ProfilingCommandsTest, givenKernelWhenProfilingCommandStartIsTakenThenTimeStampAddressIsProgrammedCorrectly, IGFX_ROCKETLAKE);
 
 HWCMDTEST_EXCLUDE_FAMILY(ProfilingCommandsTest, givenKernelWhenProfilingCommandStartIsNotTakenThenTimeStampAddressIsProgrammedCorrectly, IGFX_ROCKETLAKE);
+
+HWCMDTEST_EXCLUDE_FAMILY(BufferSetSurfaceTests, givenBufferSetSurfaceThatMemoryIsUnalignedToCachelineButReadOnlyThenL3CacheShouldBeStillOn, IGFX_ROCKETLAKE)
+
+HWCMDTEST_EXCLUDE_FAMILY(BufferSetSurfaceTests, givenAlignedCacheableReadOnlyBufferThenChoseOclBufferPolicy, IGFX_ROCKETLAKE);

opencl/test/unit_test/gen12lp/tgllp/buffer_tests_tgllp.cpp

@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2019-2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/device/device.h"
+#include "shared/source/gmm_helper/gmm_helper.h"
+#include "shared/test/unit_test/helpers/debug_manager_state_restore.h"
+
+#include "opencl/source/cl_device/cl_device.h"
+#include "opencl/source/mem_obj/buffer.h"
+#include "opencl/test/unit_test/mocks/mock_context.h"
+#include "test.h"
+
+using namespace NEO;
+
+struct BufferTestsTgllp : ::testing::Test {
+    void SetUp() override {
+        context = std::make_unique<MockContext>();
+        device = context->getDevice(0);
+    }
+
+    std::unique_ptr<MockContext> context{};
+    ClDevice *device{};
+
+    cl_int retVal = CL_SUCCESS;
+};
+
+GEN12LPTEST_F(BufferTestsTgllp, givenBufferNotReadonlyWhenProgrammingSurfaceStateThenUseL3) {
+    auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+        context.get(),
+        CL_MEM_READ_WRITE,
+        MemoryConstants::pageSize,
+        nullptr,
+        retVal));
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    typename FamilyType::RENDER_SURFACE_STATE surfaceState = {};
+    buffer->setArgStateful(&surfaceState, false, false, false, false, device->getDevice());
+
+    const auto expectedMocs = device->getGmmHelper()->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER);
+    const auto actualMocs = surfaceState.getMemoryObjectControlState();
+    EXPECT_EQ(expectedMocs, actualMocs);
+}
+
+GEN12LPTEST_F(BufferTestsTgllp, givenBufferReadonlyWhenProgrammingSurfaceStateThenUseL1) {
+    auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+        context.get(),
+        CL_MEM_READ_WRITE,
+        MemoryConstants::pageSize,
+        nullptr,
+        retVal));
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    typename FamilyType::RENDER_SURFACE_STATE surfaceState = {};
+    buffer->setArgStateful(&surfaceState, false, false, false, true, context->getDevice(0)->getDevice());
+
+    const auto expectedMocs = device->getGmmHelper()->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CONST);
+    const auto actualMocs = surfaceState.getMemoryObjectControlState();
+    EXPECT_EQ(expectedMocs, actualMocs);
+}
+
+GEN12LPTEST_F(BufferTestsTgllp, givenConstantSurfaceWhenProgrammingSurfaceStateThenUseL1) {
+    auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+        context.get(),
+        CL_MEM_READ_WRITE,
+        MemoryConstants::pageSize,
+        nullptr,
+        retVal));
+    ASSERT_EQ(CL_SUCCESS, retVal);
+    buffer->getGraphicsAllocation(0)->setAllocationType(GraphicsAllocation::AllocationType::CONSTANT_SURFACE);
+
+    typename FamilyType::RENDER_SURFACE_STATE surfaceState = {};
+    buffer->setArgStateful(&surfaceState, false, false, false, false, context->getDevice(0)->getDevice());
+
+    const auto expectedMocs = device->getGmmHelper()->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CONST);
+    const auto actualMocs = surfaceState.getMemoryObjectControlState();
+    EXPECT_EQ(expectedMocs, actualMocs);
+}
+
+GEN12LPTEST_F(BufferTestsTgllp, givenL1ForceEnabledWhenProgrammingSurfaceStateThenUseL1) {
+    DebugManagerStateRestore restore{};
+    DebugManager.flags.ForceL1Caching.set(1);
+
+    auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+        context.get(),
+        CL_MEM_READ_WRITE,
+        MemoryConstants::pageSize,
+        nullptr,
+        retVal));
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    typename FamilyType::RENDER_SURFACE_STATE surfaceState = {};
+    buffer->setArgStateful(&surfaceState, false, false, false, false, device->getDevice());
+
+    const auto expectedMocs = device->getGmmHelper()->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CONST);
+    const auto actualMocs = surfaceState.getMemoryObjectControlState();
+    EXPECT_EQ(expectedMocs, actualMocs);
+}
+
+GEN12LPTEST_F(BufferTestsTgllp, givenBufferReadonlyL1ForceDisabledWhenProgrammingSurfaceStateThenUseL3) {
+    DebugManagerStateRestore restore{};
+    DebugManager.flags.ForceL1Caching.set(0);
+
+    auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+        context.get(),
+        CL_MEM_READ_WRITE,
+        MemoryConstants::pageSize,
+        nullptr,
+        retVal));
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    typename FamilyType::RENDER_SURFACE_STATE surfaceState = {};
+    buffer->setArgStateful(&surfaceState, false, false, false, true, device->getDevice());
+
+    const auto expectedMocs = device->getGmmHelper()->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER);
+    const auto actualMocs = surfaceState.getMemoryObjectControlState();
+    EXPECT_EQ(expectedMocs, actualMocs);
+}

opencl/test/unit_test/gen12lp/tgllp/excludes_tgllp.cpp

@@ -26,3 +26,7 @@ HWCMDTEST_EXCLUDE_FAMILY(TimestampEventCreate, givenEventTimestampsWhenQueryKern
 HWCMDTEST_EXCLUDE_FAMILY(ProfilingCommandsTest, givenKernelWhenProfilingCommandStartIsTakenThenTimeStampAddressIsProgrammedCorrectly, IGFX_TIGERLAKE_LP);
 
 HWCMDTEST_EXCLUDE_FAMILY(ProfilingCommandsTest, givenKernelWhenProfilingCommandStartIsNotTakenThenTimeStampAddressIsProgrammedCorrectly, IGFX_TIGERLAKE_LP);
+
+HWCMDTEST_EXCLUDE_FAMILY(BufferSetSurfaceTests, givenBufferSetSurfaceThatMemoryIsUnalignedToCachelineButReadOnlyThenL3CacheShouldBeStillOn, IGFX_TIGERLAKE_LP)
+
+HWCMDTEST_EXCLUDE_FAMILY(BufferSetSurfaceTests, givenAlignedCacheableReadOnlyBufferThenChoseOclBufferPolicy, IGFX_TIGERLAKE_LP);

opencl/test/unit_test/test_files/igdrcl.config

@@ -27,6 +27,7 @@ AUBDumpAllocsOnEnqueueReadOnly = 0
 AUBDumpAllocsOnEnqueueSVMMemcpyOnly = 0
 AUBDumpForceAllToLocalMemory = 0
 ForceDeviceId = unk
+ForceL1Caching = -1
 SchedulerSimulationReturnInstance = 0
 SchedulerGWS = 0
 EnableExperimentalCommandBuffer = 0