From d2cbd45ca6e19df4c359c75480e6e8194be18232 Mon Sep 17 00:00:00 2001 From: juanchuletas Date: Mon, 20 Jul 2026 19:44:33 -0600 Subject: [PATCH 1/6] feature(opencl): enable bindless image support for OpenCL frontend Enable BindlessHeapsHelper initialization for the OpenCL frontend by querying hardware capability via releaseHelper instead of returning false unconditionally in getGlobalBindlessHeapConfiguration. Add opt-in bindless image creation through CL_MEM_BINDLESS_IMAGE_INTEL property. When set, Image::create allocates a bindless slot in the global Surface State Heap using the shared BindlessHeapsHelper infrastructure already used by the Level Zero frontend. Signed-off-by: juan.garcia.cpp@gmail.com --- opencl/extensions/public/cl_ext_private.h | 6 ++++++ .../source/helpers/api_specific_config_ocl.cpp | 6 +++++- .../helpers/cl_memory_properties_helpers.cpp | 9 ++++++--- opencl/source/mem_obj/image.cpp | 13 +++++++++++++ opencl/source/mem_obj/image.h | 6 ++++++ .../helpers/api_specific_config_ocl_tests.cpp | 13 +++++++++++++ .../cl_memory_properties_helpers_tests.cpp | 18 ++++++++++++++++++ .../source/helpers/memory_properties_flags.h | 1 + 8 files changed, 68 insertions(+), 4 deletions(-) diff --git a/opencl/extensions/public/cl_ext_private.h b/opencl/extensions/public/cl_ext_private.h index 7ee6d576b40a3..5c38172763f3a 100644 --- a/opencl/extensions/public/cl_ext_private.h +++ b/opencl/extensions/public/cl_ext_private.h @@ -421,4 +421,10 @@ typedef struct _cl_kernel_allocation_info_intel { // cl_device bfloat16 atomic capabilities #if !defined(CL_DEVICE_BFLOAT16_FP_ATOMIC_CAPABILITIES_EXT) #define CL_DEVICE_BFLOAT16_FP_ATOMIC_CAPABILITIES_EXT 0x10012 + +/****************************************************** + * cl_intel_bindless_images extension * + ******************************************************/ +#define CL_MEM_BINDLESS_IMAGE_INTEL 0x4220 + #endif diff --git a/opencl/source/helpers/api_specific_config_ocl.cpp b/opencl/source/helpers/api_specific_config_ocl.cpp index 495dcd7cb4a42..ae64356b674e2 100644 --- a/opencl/source/helpers/api_specific_config_ocl.cpp +++ b/opencl/source/helpers/api_specific_config_ocl.cpp @@ -9,6 +9,7 @@ #include "shared/source/device/device.h" #include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/compiler_product_helper.h" +#include "shared/source/release_helper/release_helper.h" #include "opencl/source/os_interface/ocl_reg_path.h" @@ -20,7 +21,10 @@ StackVec validClPrefixes; StackVec validClPrefixTypes; bool ApiSpecificConfig::getGlobalBindlessHeapConfiguration(const ReleaseHelper &releaseHelper) { - return false; + if (debugManager.flags.UseExternalAllocatorForSshAndDsh.get() != -1) { + return debugManager.flags.UseExternalAllocatorForSshAndDsh.get(); + } + return releaseHelper.isGlobalBindlessAllocatorEnabled(); } bool ApiSpecificConfig::getBindlessMode(const Device &device) { diff --git a/opencl/source/helpers/cl_memory_properties_helpers.cpp b/opencl/source/helpers/cl_memory_properties_helpers.cpp index 68843602c6564..61f97e36d1924 100644 --- a/opencl/source/helpers/cl_memory_properties_helpers.cpp +++ b/opencl/source/helpers/cl_memory_properties_helpers.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2024 Intel Corporation + * Copyright (C) 2021-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ bool ClMemoryPropertiesHelper::parseMemoryProperties(const cl_mem_properties_int uint64_t handleType = 0; uintptr_t hostptr = 0; std::vector devices; - + bool bindlessImage = false; if (properties != nullptr) { for (int i = 0; properties[i] != 0; i += 2) { switch (properties[i]) { @@ -73,6 +73,9 @@ bool ClMemoryPropertiesHelper::parseMemoryProperties(const cl_mem_properties_int i++; } break; + case CL_MEM_BINDLESS_IMAGE_INTEL: // Avoiding get a false when using bindless image extension + bindlessImage = true; + break; default: return false; } @@ -84,7 +87,7 @@ bool ClMemoryPropertiesHelper::parseMemoryProperties(const cl_mem_properties_int memoryProperties.handle = handle; memoryProperties.hostptr = hostptr; memoryProperties.associatedDevices = devices; - + memoryProperties.flags.bindlessImage = bindlessImage; switch (objectType) { case ClMemoryPropertiesHelper::ObjType::buffer: return isFieldValid(flags, MemObjHelper::validFlagsForBuffer) && diff --git a/opencl/source/mem_obj/image.cpp b/opencl/source/mem_obj/image.cpp index 5d2d8541f296f..769836d18465c 100644 --- a/opencl/source/mem_obj/image.cpp +++ b/opencl/source/mem_obj/image.cpp @@ -300,6 +300,19 @@ Image *Image::create(Context *context, setImageProperties(image, *imageDesc, imgInfo, parentImage, parentBuffer, hostPtrRowPitch, hostPtrSlicePitch, imageCount, hostPtrMinSize); + auto defaultRootDeviceEnv = defaultDevice->getExecutionEnvironment()->rootDeviceEnvironments[defaultRootDeviceIndex].get(); + auto bindlessHelper = defaultRootDeviceEnv->getBindlessHeapsHelper(); + if (bindlessHelper && image && memoryProperties.flags.bindlessImage) { + auto allocation = image->getGraphicsAllocation(defaultRootDeviceIndex); + auto memManager = context->getMemoryManager(); + if (memManager->allocateBindlessSlot(allocation)) { + if (allocation->getBindlessOffset() != std::numeric_limits::max()) { + image->bindlessInfo = std::make_unique(allocation->getBindlessInfo()); + image->bindlessImage = true; + } + } + } + errcodeRet = CL_SUCCESS; auto &defaultHwInfo = defaultDevice->getHardwareInfo(); if (context->isProvidingPerformanceHints()) { diff --git a/opencl/source/mem_obj/image.h b/opencl/source/mem_obj/image.h index 55b8f7d3b2b95..cc61e210a1174 100644 --- a/opencl/source/mem_obj/image.h +++ b/opencl/source/mem_obj/image.h @@ -6,6 +6,7 @@ */ #pragma once +#include "shared/source/helpers/bindless_heaps_helper.h" #include "shared/source/memory_manager/graphics_allocation.h" #include "opencl/source/helpers/surface_formats.h" @@ -205,6 +206,9 @@ class Image : public MemObj { void fillImageRegion(size_t *region) const; static bool validateHandleType(MemoryProperties &memoryProperties, UnifiedSharingMemoryDescription &extMem); + SurfaceStateInHeapInfo *getBindlessSlot() const { return bindlessInfo.get(); } + uint64_t getBindlessHandle() const { return bindlessInfo ? bindlessInfo->surfaceStateOffset : 0; } + bool isBindlessImage() const { return bindlessImage; } void setAs3DUavOrRtvImage(bool isUavOrRtv); void setIsPackedFormat(bool isPackedFormat) { this->isPackedFormat = isPackedFormat; } @@ -248,6 +252,8 @@ class Image : public MemObj { ImagePlane plane = ImagePlane::noPlane; bool is3DUAVOrRTV = false; bool isPackedFormat = false; + std::unique_ptr bindlessInfo; + bool bindlessImage = false; static bool isValidSingleChannelFormat(const cl_image_format *imageFormat); static bool isValidIntensityFormat(const cl_image_format *imageFormat); diff --git a/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp b/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp index 10547f82f8d33..362a6284061e5 100644 --- a/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp +++ b/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp @@ -8,7 +8,9 @@ #include "shared/source/helpers/api_specific_config.h" #include "shared/source/memory_manager/allocation_properties.h" #include "shared/source/memory_manager/compression_selector.h" +#include "shared/source/release_helper/release_helper.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" +#include "shared/test/common/mocks/mock_release_helper.h" #include "opencl/source/os_interface/ocl_reg_path.h" @@ -57,4 +59,15 @@ TEST(ApiSpecificConfigOclTests, WhenCheckingIfDeviceUsmPoolingIsEnabledThenRetur EXPECT_TRUE(ApiSpecificConfig::isDeviceUsmPoolingEnabled()); } +TEST(ApiSpecificConfigOclTests, WhenGettingGlobalBindlessHeapConfigurationWithDebugFlagThenReturnDebugFlagValue) { + DebugManagerStateRestore restorer; + MockReleaseHelper releaseHelper; + + debugManager.flags.UseExternalAllocatorForSshAndDsh.set(1); + EXPECT_TRUE(ApiSpecificConfig::getGlobalBindlessHeapConfiguration(releaseHelper)); + + debugManager.flags.UseExternalAllocatorForSshAndDsh.set(0); + EXPECT_FALSE(ApiSpecificConfig::getGlobalBindlessHeapConfiguration(releaseHelper)); +} + } // namespace NEO diff --git a/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp b/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp index ca4b1d3c2ee03..7c950592ae2db 100644 --- a/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp +++ b/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp @@ -14,6 +14,7 @@ #include "shared/test/common/mocks/mock_graphics_allocation.h" #include "shared/test/common/mocks/ult_device_factory.h" +#include "opencl/extensions/public/cl_ext_private.h" #include "opencl/source/helpers/cl_memory_properties_helpers.h" #include "opencl/source/mem_obj/mem_obj_helper.h" #include "opencl/test/unit_test/mocks/mock_cl_device.h" @@ -599,3 +600,20 @@ TEST_F(MemoryPropertiesHelperTests, givenSubDeviceIdWhenParsingExtraMemoryProper EXPECT_EQ(0b10u, memoryProperties.pDevice->getDeviceBitfield().to_ulong()); EXPECT_EQ(&context.pSubDevice1->getDevice(), memoryProperties.pDevice); } +TEST_F(MemoryPropertiesHelperTests, givenBindlessImagePropertyWhenParsingMemoryPropertiesForImageThenTrueIsReturnedAndFlagIsSet) { + cl_mem_properties_intel properties[] = { + CL_MEM_BINDLESS_IMAGE_INTEL, 1, + 0}; + EXPECT_TRUE(ClMemoryPropertiesHelper::parseMemoryProperties(properties, memoryProperties, flags, flagsIntel, allocflags, + ClMemoryPropertiesHelper::ObjType::image, context)); + EXPECT_TRUE(memoryProperties.flags.bindlessImage); +} + +TEST_F(MemoryPropertiesHelperTests, givenNoBindlessImagePropertyWhenParsingMemoryPropertiesForImageThenBindlessFlagIsNotSet) { + cl_mem_properties_intel properties[] = { + CL_MEM_FLAGS, CL_MEM_READ_WRITE, + 0}; + EXPECT_TRUE(ClMemoryPropertiesHelper::parseMemoryProperties(properties, memoryProperties, flags, flagsIntel, allocflags, + ClMemoryPropertiesHelper::ObjType::image, context)); + EXPECT_FALSE(memoryProperties.flags.bindlessImage); +} diff --git a/shared/source/helpers/memory_properties_flags.h b/shared/source/helpers/memory_properties_flags.h index 5611a3c206d64..4d2c03667501d 100644 --- a/shared/source/helpers/memory_properties_flags.h +++ b/shared/source/helpers/memory_properties_flags.h @@ -35,6 +35,7 @@ struct MemoryFlags { uint32_t compressedHint : 1; uint32_t uncompressedHint : 1; uint32_t ipcSupportedAllocationByDefault : 1; + uint32_t bindlessImage : 1; bool operator==(const MemoryFlags &) const = default; }; From 86ec601c04e1999d06f7da8761e44368149192f0 Mon Sep 17 00:00:00 2001 From: juanchuletas Date: Thu, 23 Jul 2026 23:04:12 -0600 Subject: [PATCH 2/6] feature(opencl): add bindless handle retrieval and enable bindless mode for supported hardware Enable getBindlessMode to query hardware capability instead of defaulting to false. Add CL_IMAGE_BINDLESS_HANDLE_INTEL query key to clGetImageInfo for retrieving the bindless handle from an image. Signed-off-by: Juan Garcia --- opencl/extensions/public/cl_ext_private.h | 1 + opencl/source/helpers/api_specific_config_ocl.cpp | 3 +-- opencl/source/mem_obj/image.cpp | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/opencl/extensions/public/cl_ext_private.h b/opencl/extensions/public/cl_ext_private.h index 5c38172763f3a..f0e25679c773a 100644 --- a/opencl/extensions/public/cl_ext_private.h +++ b/opencl/extensions/public/cl_ext_private.h @@ -426,5 +426,6 @@ typedef struct _cl_kernel_allocation_info_intel { * cl_intel_bindless_images extension * ******************************************************/ #define CL_MEM_BINDLESS_IMAGE_INTEL 0x4220 +#define CL_IMAGE_BINDLESS_HANDLE_INTEL 0x4221 #endif diff --git a/opencl/source/helpers/api_specific_config_ocl.cpp b/opencl/source/helpers/api_specific_config_ocl.cpp index ae64356b674e2..db0f61f2562d1 100644 --- a/opencl/source/helpers/api_specific_config_ocl.cpp +++ b/opencl/source/helpers/api_specific_config_ocl.cpp @@ -34,9 +34,8 @@ bool ApiSpecificConfig::getBindlessMode(const Device &device) { if (debugManager.flags.UseBindlessMode.get() != -1) { return debugManager.flags.UseBindlessMode.get(); - } else { - return false; } + return device.getReleaseHelper().isGlobalBindlessAllocatorEnabled(); } bool ApiSpecificConfig::isDeviceAllocationCacheEnabled() { diff --git a/opencl/source/mem_obj/image.cpp b/opencl/source/mem_obj/image.cpp index 769836d18465c..914e8859cf918 100644 --- a/opencl/source/mem_obj/image.cpp +++ b/opencl/source/mem_obj/image.cpp @@ -855,7 +855,11 @@ cl_int Image::getImageInfo(cl_image_info paramName, srcParamSize = sizeof(cl_uint); srcParam = &(imageDesc.num_samples); break; - + case CL_IMAGE_BINDLESS_HANDLE_INTEL: + srcParamSize = sizeof(uint64_t); + retParam = getBindlessHandle(); + srcParam = &retParam; + break; default: getOsSpecificImageInfo(paramName, &srcParamSize, &srcParam); break; From 726742106f943bb123c5bc60136b519f0917bdff Mon Sep 17 00:00:00 2001 From: juanchuletas Date: Sat, 1 Aug 2026 14:41:31 -0600 Subject: [PATCH 3/6] fix(opencl): decouple bindless heap creation from dispatch mode Remove getBindlessMode check from createBindlessHeapsHelper so the global Surface State Heap is created based on hardware capability alone. Keep getBindlessMode returning false for OpenCL to preserve normal bound-mode dispatch for all kernel arguments. This allows bindless images to explicitly allocate slots in the persistent heap via CL_MEM_BINDLESS_IMAGE_INTEL without breaking existing kernel execution. Signed-off-by: Juan Garcia --- opencl/extensions/public/cl_ext_private.h | 4 ++-- opencl/source/helpers/api_specific_config_ocl.cpp | 3 +-- .../helpers/api_specific_config_ocl_tests.cpp | 11 +++++++++++ shared/source/device/root_device.cpp | 3 +-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/opencl/extensions/public/cl_ext_private.h b/opencl/extensions/public/cl_ext_private.h index f0e25679c773a..0a3f839315f4b 100644 --- a/opencl/extensions/public/cl_ext_private.h +++ b/opencl/extensions/public/cl_ext_private.h @@ -425,7 +425,7 @@ typedef struct _cl_kernel_allocation_info_intel { /****************************************************** * cl_intel_bindless_images extension * ******************************************************/ -#define CL_MEM_BINDLESS_IMAGE_INTEL 0x4220 -#define CL_IMAGE_BINDLESS_HANDLE_INTEL 0x4221 +#define CL_MEM_BINDLESS_IMAGE_INTEL 0x10060 +#define CL_IMAGE_BINDLESS_HANDLE_INTEL 0x10061 #endif diff --git a/opencl/source/helpers/api_specific_config_ocl.cpp b/opencl/source/helpers/api_specific_config_ocl.cpp index db0f61f2562d1..f1f38ed80f844 100644 --- a/opencl/source/helpers/api_specific_config_ocl.cpp +++ b/opencl/source/helpers/api_specific_config_ocl.cpp @@ -31,11 +31,10 @@ bool ApiSpecificConfig::getBindlessMode(const Device &device) { if (device.getCompilerProductHelper().isForceBindlessRequired(device.getHardwareInfo())) { return true; } - if (debugManager.flags.UseBindlessMode.get() != -1) { return debugManager.flags.UseBindlessMode.get(); } - return device.getReleaseHelper().isGlobalBindlessAllocatorEnabled(); + return false; } bool ApiSpecificConfig::isDeviceAllocationCacheEnabled() { diff --git a/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp b/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp index 362a6284061e5..9ea5757598b5a 100644 --- a/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp +++ b/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp @@ -10,7 +10,9 @@ #include "shared/source/memory_manager/compression_selector.h" #include "shared/source/release_helper/release_helper.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" +#include "shared/test/common/mocks/mock_device.h" #include "shared/test/common/mocks/mock_release_helper.h" +#include "shared/test/common/mocks/ult_device_factory.h" #include "opencl/source/os_interface/ocl_reg_path.h" @@ -70,4 +72,13 @@ TEST(ApiSpecificConfigOclTests, WhenGettingGlobalBindlessHeapConfigurationWithDe EXPECT_FALSE(ApiSpecificConfig::getGlobalBindlessHeapConfiguration(releaseHelper)); } +TEST(ApiSpecificConfigOclTests, WhenGettingBindlessModeThenQueryHardwareCapability) { + DebugManagerStateRestore restorer; + UltDeviceFactory deviceFactory{1, 0}; + auto *device = deviceFactory.rootDevices[0]; + + debugManager.flags.UseBindlessMode.set(-1); + auto result = ApiSpecificConfig::getBindlessMode(static_cast(*device)); + EXPECT_EQ(result, device->getReleaseHelper().isGlobalBindlessAllocatorEnabled()); +} } // namespace NEO diff --git a/shared/source/device/root_device.cpp b/shared/source/device/root_device.cpp index 4174d07f0c02e..e4325f61a2695 100644 --- a/shared/source/device/root_device.cpp +++ b/shared/source/device/root_device.cpp @@ -54,8 +54,7 @@ void RootDevice::createBindlessHeapsHelper() { EnvironmentVariableReader envReader; bool disableGlobalBindless = envReader.getSetting("NEO_L0_SYSMAN_NO_CONTEXT_MODE", false); - - if (!disableGlobalBindless && ApiSpecificConfig::getGlobalBindlessHeapConfiguration(this->getReleaseHelper()) && ApiSpecificConfig::getBindlessMode(*this)) { + if (!disableGlobalBindless && ApiSpecificConfig::getGlobalBindlessHeapConfiguration(this->getReleaseHelper())) { this->executionEnvironment->rootDeviceEnvironments[getRootDeviceIndex()]->createBindlessHeapsHelper(this, getNumGenericSubDevices() > 1); } } From 647fcb81dadd43b7596ca3eaad9cf9f260167a23 Mon Sep 17 00:00:00 2001 From: juanchuletas Date: Mon, 3 Aug 2026 16:03:25 -0600 Subject: [PATCH 4/6] fix(opencl): complete bindless image dispatch path Kernel::patchBindlessSurfaceStatesInCrossThreadData ran on every dispatch and unconditionally re-patched the cross-thread-data offset for any explicit arg with a valid bindless offset, using the local per-dispatch Surface State Heap indexing scheme. For bindless images this was overwriting the offset that setArgImageWithMipLevel had already patched to point at the image's persistent slot in the global SSH, so the shader always read whatever (uninitialized) data happened to sit in the local SSH slot instead of the actual image descriptor. Do not repatch image args whose bound cl_mem is a genuine bindless image (CL_MEM_BINDLESS_IMAGE_INTEL), since those already carry the correct address into their own persistent slot. Signed-off-by: Juan Garcia --- opencl/source/kernel/kernel.cpp | 40 ++++++++++++++++--- opencl/source/program/build.cpp | 9 ++++- .../compiler_interface/compiler_options.h | 2 + 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/opencl/source/kernel/kernel.cpp b/opencl/source/kernel/kernel.cpp index 238089c2ae8bf..6e7c799129867 100644 --- a/opencl/source/kernel/kernel.cpp +++ b/opencl/source/kernel/kernel.cpp @@ -29,6 +29,7 @@ #include "shared/source/helpers/ptr_math.h" #include "shared/source/helpers/simd_helper.h" #include "shared/source/helpers/surface_format_info.h" +#include "shared/source/indirect_heap/indirect_heap.h" #include "shared/source/kernel/local_ids_cache.h" #include "shared/source/memory_manager/compression_selector.h" #include "shared/source/memory_manager/memory_manager.h" @@ -1334,7 +1335,17 @@ void Kernel::makeResident(CommandStreamReceiver &commandStreamReceiver) { pageFaultManager->moveAllocationsWithinUMAllocsManagerToGpuDomain(this->getContext().getSVMAllocsManager()); } makeArgsResident(commandStreamReceiver); - + auto bindlessHelper = getDevice().getDevice().getBindlessHeapsHelper(); + if (bindlessHelper) { + for (auto heapType : {NEO::BindlessHeapsHelper::specialSsh, + NEO::BindlessHeapsHelper::globalSsh, + NEO::BindlessHeapsHelper::globalDsh}) { + auto heap = bindlessHelper->getHeap(heapType); + if (heap) { + commandStreamReceiver.makeResident(*heap->getGraphicsAllocation()); + } + } + } auto kernelIsaAllocation = this->kernelInfo.getIsaGraphicsAllocation(); if (kernelIsaAllocation) { commandStreamReceiver.makeResident(*kernelIsaAllocation); @@ -1693,7 +1704,19 @@ cl_int Kernel::setArgImageWithMipLevel(uint32_t argIndex, imageFromBufferArgsCount += (pImage->isImageFromBuffer() ? 1 : 0) - (wasImageFromBuffer ? 1 : 0); void *surfaceState = nullptr; - if (isValidOffset(argAsImg.bindless)) { + if (isValidOffset(argAsImg.bindless) && pImage->isBindlessImage()) { + auto bindlessSlot = pImage->getBindlessSlot(); + if (bindlessSlot && bindlessSlot->ssPtr) { + surfaceState = bindlessSlot->ssPtr; + + auto &gfxCoreHelper = this->getGfxCoreHelper(); + auto patchLocation = ptrOffset(getCrossThreadData(), argAsImg.bindless); + uint64_t patchValue = gfxCoreHelper.getBindlessSurfaceExtendedMessageDescriptorValue( + static_cast(bindlessSlot->surfaceStateOffset)); + uint32_t patchSize = NEO::isUndefinedOffset(argAsImg.size) ? 0 : argAsImg.size; + patchWithRequiredSize(reinterpret_cast(patchLocation), patchSize, patchValue); + } + } else if (isValidOffset(argAsImg.bindless)) { auto ssIndex = getSurfaceStateIndexForBindlessOffset(argAsImg.bindless); if (ssIndex < std::numeric_limits::max()) { auto &gfxCoreHelper = this->getGfxCoreHelper(); @@ -1705,10 +1728,8 @@ cl_int Kernel::setArgImageWithMipLevel(uint32_t argIndex, surfaceState = ptrOffset(getSurfaceStateHeap(), argAsImg.bindful); } - // Sets SS structure UNRECOVERABLE_IF(surfaceState == nullptr); pImage->setImageArg(surfaceState, arg.getExtendedTypeInfo().isMediaBlockImage, mipLevel, rootDeviceIndex); - auto &imageDesc = pImage->getImageDesc(); auto &imageFormat = pImage->getImageFormat(); auto graphicsAllocation = pImage->getGraphicsAllocation(rootDeviceIndex); @@ -2048,13 +2069,22 @@ void Kernel::patchBindlessSurfaceStatesInCrossThreadData(uint64_t bindlessSurfac auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(); auto *crossThreadDataPtr = reinterpret_cast(getCrossThreadData()); - for (auto &arg : kernelInfo.kernelDescriptor.payloadMappings.explicitArgs) { + const auto &explicitArgs = kernelInfo.kernelDescriptor.payloadMappings.explicitArgs; + for (size_t argIndex = 0; argIndex < explicitArgs.size(); argIndex++) { + const auto &arg = explicitArgs[argIndex]; auto offset = NEO::undefined; if (arg.type == NEO::ArgDescriptor::argTPointer) { offset = arg.as().bindless; } else if (arg.type == NEO::ArgDescriptor::argTImage) { offset = arg.as().bindless; + if (NEO::isValidOffset(offset)) { + auto clMem = static_cast(kernelArguments[argIndex].object); + auto pImage = castToObject(clMem); + if (pImage && pImage->isBindlessImage()) { + continue; + } + } } else { continue; } diff --git a/opencl/source/program/build.cpp b/opencl/source/program/build.cpp index 99e7ed863d0db..73b6c51a192a3 100644 --- a/opencl/source/program/build.cpp +++ b/opencl/source/program/build.cpp @@ -55,7 +55,14 @@ cl_int Program::build( options = ""; } } - + const bool hasBindlessImages = CompilerOptions::extract(CompilerOptions::bindlessImages, options); + const bool hasBindlessAdvancedMode = CompilerOptions::extract(CompilerOptions::bindlessAdvancedMode, options); + if (hasBindlessImages) { + CompilerOptions::concatenateAppend(internalOptions, CompilerOptions::bindlessImages); + } + if (hasBindlessAdvancedMode) { + CompilerOptions::concatenateAppend(internalOptions, CompilerOptions::bindlessAdvancedMode); + } const bool shouldSuppressRebuildWarning{CompilerOptions::extract(CompilerOptions::noRecompiledFromIr, options)}; extractInternalOptions(options, internalOptions); CompilerOptions::applyAdditionalApiOptions(options); diff --git a/shared/source/compiler_interface/compiler_options.h b/shared/source/compiler_interface/compiler_options.h index 3fe042adcf7cb..b4c6f256a0b17 100644 --- a/shared/source/compiler_interface/compiler_options.h +++ b/shared/source/compiler_interface/compiler_options.h @@ -32,6 +32,8 @@ inline constexpr ConstStringRef createLibrary = "-create-library"; inline constexpr ConstStringRef generateDebugInfo = "-g"; inline constexpr ConstStringRef generateSourcePath = "-s"; inline constexpr ConstStringRef bindlessMode = "-cl-intel-use-bindless-mode -cl-intel-use-bindless-advanced-mode"; +inline constexpr ConstStringRef bindlessImages = "-cl-intel-use-bindless-images"; +inline constexpr ConstStringRef bindlessAdvancedMode = "-cl-intel-use-bindless-advanced-mode"; inline constexpr ConstStringRef uniformWorkgroupSize = "-cl-uniform-work-group-size"; inline constexpr ConstStringRef forceEmuInt32DivRem = "-cl-intel-force-emu-int32divrem"; inline constexpr ConstStringRef forceEmuInt32DivRemSP = "-cl-intel-force-emu-sp-int32divrem"; From aa8de09c29f4daba4355669e1d2798b12c7e221a Mon Sep 17 00:00:00 2001 From: juanchuletas Date: Thu, 20 Aug 2026 17:31:16 -0600 Subject: [PATCH 5/6] Populate the SSH at bindless image creation setImageArg should immediately be called, after allocateBindlessSlot in order to fill the surface state descriptor at image creation time, by doing this real bindless access is enabled using handles without kernel argument binding. Signed-off-by: Juan Garcia --- opencl/source/mem_obj/image.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opencl/source/mem_obj/image.cpp b/opencl/source/mem_obj/image.cpp index 914e8859cf918..86fa115ad6b57 100644 --- a/opencl/source/mem_obj/image.cpp +++ b/opencl/source/mem_obj/image.cpp @@ -309,6 +309,9 @@ Image *Image::create(Context *context, if (allocation->getBindlessOffset() != std::numeric_limits::max()) { image->bindlessInfo = std::make_unique(allocation->getBindlessInfo()); image->bindlessImage = true; + if (image->bindlessInfo->ssPtr) { + image->setImageArg(image->bindlessInfo->ssPtr, false, 0, defaultRootDeviceIndex); + } } } } From 442e84c8146f5750239ddca7277465487e222d51 Mon Sep 17 00:00:00 2001 From: juanchuletas Date: Thu, 1 Oct 2026 14:25:28 -0600 Subject: [PATCH 6/6] Create BindlessHeapsHelper only when an OpenCL bindless image is requested. Allocate the persistent bindless image SSH through a dedicated allocation type, keeping ordinary OpenCL linear stream allocations unchanged. Preserve existing Level Zero bindless heap allocation behavior. --- .../source/helpers/api_specific_config_l0.cpp | 4 ++ opencl/source/command_queue/enqueue_common.h | 1 + .../helpers/api_specific_config_ocl.cpp | 11 +++--- .../helpers/hardware_commands_helper_base.inl | 4 +- opencl/source/helpers/task_information.cpp | 1 + opencl/source/kernel/kernel.cpp | 7 +++- opencl/source/kernel/kernel.h | 2 + opencl/source/mem_obj/image.cpp | 8 ++++ opencl/source/program/build.cpp | 4 ++ opencl/source/program/program.h | 6 +++ .../helpers/api_specific_config_ocl_tests.cpp | 25 +++++-------- opencl/test/unit_test/kernel/kernel_tests.cpp | 12 ++++-- .../os_interface/device_factory_tests.cpp | 7 +++- .../command_stream_receiver_hw.h | 4 +- .../command_stream_receiver_hw_base.inl | 37 ++++++++++++++----- .../source/command_stream/csr_definitions.h | 1 + shared/source/device/root_device.cpp | 6 ++- .../root_device_environment.cpp | 6 ++- .../root_device_environment.h | 1 + .../gmm_helper/cache_settings_helper.cpp | 2 + shared/source/helpers/api_specific_config.h | 1 + .../helpers/app_resource_classification.h | 1 + shared/source/helpers/app_resource_helper.cpp | 2 + .../source/helpers/bindless_heaps_helper.cpp | 29 ++++++++++----- shared/source/helpers/bindless_heaps_helper.h | 3 +- shared/source/helpers/heap_assigner.cpp | 3 ++ .../source/memory_manager/allocation_type.h | 1 + .../definitions/storage_info.cpp | 2 + .../memory_manager/graphics_allocation.h | 1 + .../source/memory_manager/memory_manager.cpp | 2 + shared/source/utilities/logger.cpp | 2 + .../unit_test/api_specific_config_ult.cpp | 6 ++- .../unit_test/gmm_helper/gmm_helper_tests.cpp | 8 +++- .../app_resource_classification_tests.cpp | 3 +- .../unit_test/helpers/app_resource_tests.cpp | 1 + ...drm_memory_manager_bindless_heap_tests.cpp | 7 ++++ 36 files changed, 167 insertions(+), 54 deletions(-) diff --git a/level_zero/core/source/helpers/api_specific_config_l0.cpp b/level_zero/core/source/helpers/api_specific_config_l0.cpp index e28b2238ef749..01e281000f407 100644 --- a/level_zero/core/source/helpers/api_specific_config_l0.cpp +++ b/level_zero/core/source/helpers/api_specific_config_l0.cpp @@ -115,5 +115,9 @@ bool ApiSpecificConfig::isGlobalStatelessEnabled(const RootDeviceEnvironment &ro bool ApiSpecificConfig::isUpdateTagFromWaitEnabledForHeapless() { return false; } +bool ApiSpecificConfig::createBindlessHeapsHelper(const Device &device) { + return getGlobalBindlessHeapConfiguration(device.getHardwareInfo()) && getBindlessMode(device); +} + } // namespace NEO diff --git a/opencl/source/command_queue/enqueue_common.h b/opencl/source/command_queue/enqueue_common.h index 67116fc75167e..43c268a3fce4d 100644 --- a/opencl/source/command_queue/enqueue_common.h +++ b/opencl/source/command_queue/enqueue_common.h @@ -937,6 +937,7 @@ CompletionStamp CommandQueueHw::enqueueNonBlocked( isDcFlushRequiredOnStallingCommandsOnNextFlush() // isDcFlushRequiredOnStallingCommandsOnNextFlush ); + dispatchFlags.useBindlessImages = kernel->usesBindlessImages(); dispatchFlags.isWalkerWithProfilingEnqueued = getAndClearIsWalkerWithProfilingEnqueued(); dispatchFlags.pipelineSelectArgs.systolicPipelineSelectMode = systolicPipelineSelectMode; uint32_t lws[3] = {static_cast(multiDispatchInfo.begin()->getLocalWorkgroupSize().x), static_cast(multiDispatchInfo.begin()->getLocalWorkgroupSize().y), static_cast(multiDispatchInfo.begin()->getLocalWorkgroupSize().z)}; diff --git a/opencl/source/helpers/api_specific_config_ocl.cpp b/opencl/source/helpers/api_specific_config_ocl.cpp index 300855a10ef79..82f7dfa9023d5 100644 --- a/opencl/source/helpers/api_specific_config_ocl.cpp +++ b/opencl/source/helpers/api_specific_config_ocl.cpp @@ -20,11 +20,8 @@ namespace NEO { StackVec validClPrefixes; StackVec validClPrefixTypes; -bool ApiSpecificConfig::getGlobalBindlessHeapConfiguration(const HardwareInfo &hwInfo) { - if (debugManager.flags.UseExternalAllocatorForSshAndDsh.get() != -1) { - return debugManager.flags.UseExternalAllocatorForSshAndDsh.get(); - } - return hwInfo.caps.globalBindlessAllocatorEnabled; +bool ApiSpecificConfig::getGlobalBindlessHeapConfiguration(const HardwareInfo &) { + return false; } bool ApiSpecificConfig::getBindlessMode(const Device &device) { @@ -98,4 +95,8 @@ bool ApiSpecificConfig::isUpdateTagFromWaitEnabledForHeapless() { return true; } +bool ApiSpecificConfig::createBindlessHeapsHelper(const Device &) { + return false; +} + } // namespace NEO diff --git a/opencl/source/helpers/hardware_commands_helper_base.inl b/opencl/source/helpers/hardware_commands_helper_base.inl index 3758eb4f33224..22032363278d5 100644 --- a/opencl/source/helpers/hardware_commands_helper_base.inl +++ b/opencl/source/helpers/hardware_commands_helper_base.inl @@ -13,6 +13,7 @@ #include "shared/source/device/device.h" #include "shared/source/helpers/address_patch.h" #include "shared/source/helpers/aligned_memory.h" +#include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/basic_math.h" #include "shared/source/helpers/gfx_core_helper.h" #include "shared/source/helpers/hw_info.h" @@ -260,9 +261,10 @@ size_t HardwareCommandsHelper::sendIndirectState( const auto &samplerTable = kernelInfo.kernelDescriptor.payloadMappings.samplerTable; if (isValidOffset(samplerTable.tableOffset) && isValidOffset(samplerTable.borderColor)) { samplerCount = samplerTable.numSamplers; + auto *samplerHeapHelper = ApiSpecificConfig::getBindlessMode(device) ? device.getBindlessHeapsHelper() : nullptr; samplerStateOffset = EncodeStates::copySamplerState(&dsh, samplerTable.tableOffset, samplerCount, samplerTable.borderColor, - kernel.getDynamicStateHeap(), device.getBindlessHeapsHelper(), + kernel.getDynamicStateHeap(), samplerHeapHelper, device.getRootDeviceEnvironment()); if constexpr (heaplessModeEnabled) { uint64_t bindlessSamplerStateAddress = samplerStateOffset; diff --git a/opencl/source/helpers/task_information.cpp b/opencl/source/helpers/task_information.cpp index d37ac6da0ba4d..a9e3a084cb926 100644 --- a/opencl/source/helpers/task_information.cpp +++ b/opencl/source/helpers/task_information.cpp @@ -216,6 +216,7 @@ CompletionStamp &CommandComputeKernel::submit(TaskCountType taskLevel, bool term commandQueue.isStallingCommandsOnNextFlushRequired(), // isStallingCommandsOnNextFlushRequired commandQueue.isDcFlushRequiredOnStallingCommandsOnNextFlush() // isDcFlushRequiredOnStallingCommandsOnNextFlush ); + dispatchFlags.useBindlessImages = kernel->usesBindlessImages(); if (commandQueue.getContext().getRootDeviceIndices().size() > 1) { eventsRequest.fillCsrDependenciesForRootDevices(dispatchFlags.csrDependencies, commandStreamReceiver); } diff --git a/opencl/source/kernel/kernel.cpp b/opencl/source/kernel/kernel.cpp index 1307e07a4c934..c731168d36b6a 100644 --- a/opencl/source/kernel/kernel.cpp +++ b/opencl/source/kernel/kernel.cpp @@ -18,6 +18,7 @@ #include "shared/source/gmm_helper/resource_info.h" #include "shared/source/helpers/address_patch.h" #include "shared/source/helpers/aligned_memory.h" +#include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/basic_math.h" #include "shared/source/helpers/bindless_heaps_helper.h" #include "shared/source/helpers/debug_helpers.h" @@ -147,7 +148,7 @@ void Kernel::patchWithImplicitSurface(uint64_t ptrToPatchInCrossThreadData, Grap void *surfaceState = nullptr; auto surfaceStateSize = gfxCoreHelper.getBindlessSurfaceStateSlotSize(); - if (clDevice.getDevice().getBindlessHeapsHelper()) { + if (clDevice.getDevice().getBindlessHeapsHelper() && ApiSpecificConfig::getBindlessMode(clDevice.getDevice())) { auto &ssInHeap = allocation.getBindlessInfo(); surfaceState = ssInHeap.ssPtr; auto patchLocation = ptrOffset(crossThreadData, arg.bindless); @@ -1533,6 +1534,10 @@ void Kernel::getAllocationsInfo(std::vector &al } } +bool Kernel::usesBindlessImages() const { + return program->usesBindlessImages(clDevice.getRootDeviceIndex()); +} + cl_int Kernel::setArgLocal(uint32_t argIndexIn, size_t argSize, const void *argVal) { diff --git a/opencl/source/kernel/kernel.h b/opencl/source/kernel/kernel.h index ebc2a6c095a6c..5a07b170abc8f 100644 --- a/opencl/source/kernel/kernel.h +++ b/opencl/source/kernel/kernel.h @@ -391,6 +391,8 @@ class Kernel : public ReferenceTrackedObject, NEO::NonCopyableAndNonMova MOCKABLE_VIRTUAL void getAllocationsInfo(std::vector &allocationsInfo) const; + bool usesBindlessImages() const; + protected: Kernel(Program *programArg, const KernelInfo &kernelInfo, ClDevice &clDevice); diff --git a/opencl/source/mem_obj/image.cpp b/opencl/source/mem_obj/image.cpp index ee77da19e0e3f..9348acff9e505 100644 --- a/opencl/source/mem_obj/image.cpp +++ b/opencl/source/mem_obj/image.cpp @@ -302,6 +302,14 @@ Image *Image::create(Context *context, setImageProperties(image, *imageDesc, imgInfo, parentImage, parentBuffer, hostPtrRowPitch, hostPtrSlicePitch, imageCount, hostPtrMinSize); auto defaultRootDeviceEnv = defaultDevice->getExecutionEnvironment()->rootDeviceEnvironments[defaultRootDeviceIndex].get(); + + if (image && memoryProperties.flags.bindlessImage && + defaultRootDeviceEnv->getBindlessHeapsHelper() == nullptr) { + auto *rootDevice = defaultDevice->getRootDevice(); + defaultRootDeviceEnv->createBindlessHeapsHelper( + rootDevice, rootDevice->getNumGenericSubDevices() > 1); + } + auto bindlessHelper = defaultRootDeviceEnv->getBindlessHeapsHelper(); if (bindlessHelper && image && memoryProperties.flags.bindlessImage) { auto allocation = image->getGraphicsAllocation(defaultRootDeviceIndex); diff --git a/opencl/source/program/build.cpp b/opencl/source/program/build.cpp index 73b6c51a192a3..9165c381e94bf 100644 --- a/opencl/source/program/build.cpp +++ b/opencl/source/program/build.cpp @@ -57,6 +57,7 @@ cl_int Program::build( } const bool hasBindlessImages = CompilerOptions::extract(CompilerOptions::bindlessImages, options); const bool hasBindlessAdvancedMode = CompilerOptions::extract(CompilerOptions::bindlessAdvancedMode, options); + if (hasBindlessImages) { CompilerOptions::concatenateAppend(internalOptions, CompilerOptions::bindlessImages); } @@ -140,6 +141,9 @@ cl_int Program::build( } this->replaceDeviceBinary(std::move(compilerOuput.deviceBinary.mem), compilerOuput.deviceBinary.size, clDevice->getRootDeviceIndex()); phaseReached[clDevice->getRootDeviceIndex()] = BuildPhase::binaryCreation; + + this->buildInfos[clDevice->getRootDeviceIndex()].useBindlessImages = hasBindlessImages; + } if (retVal != CL_SUCCESS) { break; diff --git a/opencl/source/program/program.h b/opencl/source/program/program.h index 7b25d64920e91..b4ab89d0219e8 100644 --- a/opencl/source/program/program.h +++ b/opencl/source/program/program.h @@ -193,6 +193,10 @@ class Program : public BaseObject<_cl_program> { return isSpirV; } + bool usesBindlessImages(uint32_t rootDeviceIndex) const { + return buildInfos[rootDeviceIndex].useBindlessImages; + } + void freeGlobalBufferAllocation(std::unique_ptr &buffer); NEO::SharedPoolAllocation *getConstantSurface(uint32_t rootDeviceIndex) const; @@ -381,6 +385,8 @@ class Program : public BaseObject<_cl_program> { std::unique_ptr debugData; size_t debugDataSize = 0U; size_t kernelMiscInfoPos = std::string::npos; + + bool useBindlessImages = false; }; std::vector buildInfos; diff --git a/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp b/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp index 9ea5757598b5a..93cdc9781cf47 100644 --- a/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp +++ b/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp @@ -6,12 +6,10 @@ */ #include "shared/source/helpers/api_specific_config.h" +#include "shared/source/helpers/hw_info.h" #include "shared/source/memory_manager/allocation_properties.h" #include "shared/source/memory_manager/compression_selector.h" -#include "shared/source/release_helper/release_helper.h" -#include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/mocks/mock_device.h" -#include "shared/test/common/mocks/mock_release_helper.h" #include "shared/test/common/mocks/ult_device_factory.h" #include "opencl/source/os_interface/ocl_reg_path.h" @@ -61,24 +59,19 @@ TEST(ApiSpecificConfigOclTests, WhenCheckingIfDeviceUsmPoolingIsEnabledThenRetur EXPECT_TRUE(ApiSpecificConfig::isDeviceUsmPoolingEnabled()); } -TEST(ApiSpecificConfigOclTests, WhenGettingGlobalBindlessHeapConfigurationWithDebugFlagThenReturnDebugFlagValue) { - DebugManagerStateRestore restorer; - MockReleaseHelper releaseHelper; - - debugManager.flags.UseExternalAllocatorForSshAndDsh.set(1); - EXPECT_TRUE(ApiSpecificConfig::getGlobalBindlessHeapConfiguration(releaseHelper)); +TEST(ApiSpecificConfigOclTests, WhenGettingGlobalBindlessHeapConfigurationThenReturnFalse) { + UltDeviceFactory deviceFactory{1, 0}; + auto *device = deviceFactory.rootDevices[0]; + auto &hwInfo = device->getHardwareInfo(); - debugManager.flags.UseExternalAllocatorForSshAndDsh.set(0); - EXPECT_FALSE(ApiSpecificConfig::getGlobalBindlessHeapConfiguration(releaseHelper)); + EXPECT_FALSE(ApiSpecificConfig::getGlobalBindlessHeapConfiguration(hwInfo)); } -TEST(ApiSpecificConfigOclTests, WhenGettingBindlessModeThenQueryHardwareCapability) { - DebugManagerStateRestore restorer; +TEST(ApiSpecificConfigOclTests, WhenCreatingBindlessHeapsHelperThenReturnFalse) { UltDeviceFactory deviceFactory{1, 0}; auto *device = deviceFactory.rootDevices[0]; - debugManager.flags.UseBindlessMode.set(-1); - auto result = ApiSpecificConfig::getBindlessMode(static_cast(*device)); - EXPECT_EQ(result, device->getReleaseHelper().isGlobalBindlessAllocatorEnabled()); + auto result = ApiSpecificConfig::createBindlessHeapsHelper(*device); + EXPECT_FALSE(result); } } // namespace NEO diff --git a/opencl/test/unit_test/kernel/kernel_tests.cpp b/opencl/test/unit_test/kernel/kernel_tests.cpp index f21a77425be46..50401dd546bd7 100644 --- a/opencl/test/unit_test/kernel/kernel_tests.cpp +++ b/opencl/test/unit_test/kernel/kernel_tests.cpp @@ -484,6 +484,7 @@ TEST_F(BindlessKernelTests, givenBindlessKernelWhenPatchingCrossThreadDataThenCo pProgram->mockKernelInfo.kernelDescriptor.payloadMappings.implicitArgs.globalConstantsSurfaceAddress.bindless = 4 * sizeof(uint64_t); MockKernel mockKernel(pProgram, pProgram->mockKernelInfo, *pClDevice); + mockKernel.kernelArguments.resize(pProgram->mockKernelInfo.kernelDescriptor.payloadMappings.explicitArgs.size()); pProgram->mockKernelInfo.kernelDescriptor.initBindlessOffsetToSurfaceState(); @@ -493,7 +494,7 @@ TEST_F(BindlessKernelTests, givenBindlessKernelWhenPatchingCrossThreadDataThenCo const uint64_t baseAddress = 0x1000; auto &gfxCoreHelper = pClDevice->getGfxCoreHelper(); - auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(pClDevice->getDevice().getRootDeviceEnvironment()); + auto surfaceStateSize = gfxCoreHelper.getBindlessSurfaceStateSlotSize(); auto patchValue1 = gfxCoreHelper.getBindlessSurfaceExtendedMessageDescriptorValue(static_cast(baseAddress)); auto patchValue2 = gfxCoreHelper.getBindlessSurfaceExtendedMessageDescriptorValue(static_cast(baseAddress + 1 * surfaceStateSize)); @@ -540,6 +541,7 @@ TEST_F(BindlessKernelTests, givenBindlessKernelWhenPatchBindlessSurfaceStatesInC pProgram->mockKernelInfo.kernelDescriptor.payloadMappings.implicitArgs.globalConstantsSurfaceAddress.bindless = 4 * sizeof(uint64_t); MockKernel mockKernel(pProgram, pProgram->mockKernelInfo, *pClDevice); + mockKernel.kernelArguments.resize(pProgram->mockKernelInfo.kernelDescriptor.payloadMappings.explicitArgs.size()); pProgram->mockKernelInfo.kernelDescriptor.initBindlessOffsetToSurfaceState(); @@ -551,7 +553,7 @@ TEST_F(BindlessKernelTests, givenBindlessKernelWhenPatchBindlessSurfaceStatesInC ASSERT_TRUE(baseAddress > std::numeric_limits::max()); auto &gfxCoreHelper = pClDevice->getGfxCoreHelper(); - auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(pClDevice->getDevice().getRootDeviceEnvironment()); + auto surfaceStateSize = gfxCoreHelper.getBindlessSurfaceStateSlotSize(); auto bindlessSufaceState1Address = baseAddress; auto bindlessSufaceState2Address = baseAddress + 2 * surfaceStateSize; @@ -3393,8 +3395,7 @@ HWTEST_F(KernelTest, givenBindlessArgBufferWhenPatchWithImplicitSurfaceThenSurfa kernel.mockKernel->patchWithImplicitSurface(castToUint64(&crossThreadData), mockAllocation, kernel.kernelInfo.argAsPtr(0)); const auto &gfxCoreHelper = device->getGfxCoreHelper(); - const auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(device->getRootDeviceEnvironment()); - + const auto surfaceStateSize = gfxCoreHelper.getBindlessSurfaceStateSlotSize(); const auto ssIndex = kernel.kernelInfo.kernelDescriptor.bindlessArgsMap.find(bindlessOffset)->second; const auto ssOffset = ssIndex * surfaceStateSize; @@ -3436,6 +3437,9 @@ HWTEST_F(KernelTest, givenBindlessArgBufferAndNotInitializedBindlessOffsetToSurf } HWTEST_F(KernelTest, givenBindlessHeapsHelperAndBindlessArgBufferWhenPatchWithImplicitSurfaceThenCrossThreadDataIsPatchedAndSurfaceStateIsEncoded) { + DebugManagerStateRestore restore; + debugManager.flags.UseBindlessMode.set(1); + auto device = clUniquePtr(new MockClDevice(MockDevice::createWithNewExecutionEnvironment(defaultHwInfo.get()))); auto &neoDevice = device->getDevice(); diff --git a/opencl/test/unit_test/os_interface/device_factory_tests.cpp b/opencl/test/unit_test/os_interface/device_factory_tests.cpp index 5a3c7fb1bddd0..6279da4263cc6 100644 --- a/opencl/test/unit_test/os_interface/device_factory_tests.cpp +++ b/opencl/test/unit_test/os_interface/device_factory_tests.cpp @@ -21,6 +21,7 @@ #include "shared/test/common/mocks/mock_execution_environment.h" #include "shared/test/common/mocks/mock_io_functions.h" #include "shared/test/common/mocks/mock_memory_manager.h" +#include "shared/test/common/mocks/mock_memory_operations_handler.h" #include "shared/test/common/mocks/mock_product_helper.h" #include "shared/test/common/mocks/ult_device_factory.h" #include "shared/test/common/test_macros/hw_test.h" @@ -283,6 +284,8 @@ TEST(DeviceFactory, givenCreateMultipleRootDevicesWhenCreateDevicesIsCalledThenV hwInfo[i] = *NEO::defaultHwInfo.get(); executionEnvironment->rootDeviceEnvironments[i]->setHwInfoAndInitHelpers(&hwInfo[i]); executionEnvironment->rootDeviceEnvironments[i]->initGmm(); + executionEnvironment->rootDeviceEnvironments[i]->memoryOperationsInterface = + std::make_unique(); } executionEnvironment->rootDeviceEnvironments[0]->getMutableHardwareInfo()->capabilityTable.isIntegratedDevice = true; executionEnvironment->rootDeviceEnvironments[1]->getMutableHardwareInfo()->capabilityTable.isIntegratedDevice = true; @@ -293,7 +296,9 @@ TEST(DeviceFactory, givenCreateMultipleRootDevicesWhenCreateDevicesIsCalledThenV executionEnvironment->rootDeviceEnvironments[6]->getMutableHardwareInfo()->capabilityTable.isIntegratedDevice = true; executionEnvironment->rootDeviceEnvironments[7]->getMutableHardwareInfo()->capabilityTable.isIntegratedDevice = false; auto devices = DeviceFactory::createDevices(*executionEnvironment); - for (auto iterator = 0u; iterator < 8; iterator++) { + ASSERT_EQ(numRootDevices, devices.size()); + for (auto iterator = 0u; iterator < numRootDevices; iterator++) { + ASSERT_NE(nullptr, devices[iterator]); EXPECT_EQ(iterator, devices[iterator]->getRootDeviceIndex()); } } diff --git a/shared/source/command_stream/command_stream_receiver_hw.h b/shared/source/command_stream/command_stream_receiver_hw.h index 7a77544776dbe..cf95745b6a221 100644 --- a/shared/source/command_stream/command_stream_receiver_hw.h +++ b/shared/source/command_stream/command_stream_receiver_hw.h @@ -281,7 +281,8 @@ class CommandStreamReceiverHw : public CommandStreamReceiver { LinearStream &csrCommandStream, bool dispatchBindingTableCommand, bool areMultipleSubDevicesInContext, - bool setGeneralStateBaseAddress); + bool setGeneralStateBaseAddress, + bool useBindlessImages); inline void emitTagUpdateWithoutDCFlush(LinearStream &commandStream); @@ -372,6 +373,7 @@ class CommandStreamReceiverHw : public CommandStreamReceiver { CsrSizeRequestFlags csrSizeRequestFlags = {}; bool wasSubmittedToSingleSubdevice = false; + bool lastUseGlobalImageDescriptors = false; std::unique_ptr>> directSubmission; std::unique_ptr>> blitterDirectSubmission; diff --git a/shared/source/command_stream/command_stream_receiver_hw_base.inl b/shared/source/command_stream/command_stream_receiver_hw_base.inl index 6e64adda219c5..ed8241ca1aacb 100644 --- a/shared/source/command_stream/command_stream_receiver_hw_base.inl +++ b/shared/source/command_stream/command_stream_receiver_hw_base.inl @@ -1689,7 +1689,10 @@ inline void CommandStreamReceiverHw::programStateBaseAddress(const In bool stateBaseAddressDirty) { const auto bindlessHeapsHelper = device.getBindlessHeapsHelper(); - const bool useGlobalHeaps = bindlessHeapsHelper != nullptr; + + const bool useGlobalHeaps = bindlessHeapsHelper != nullptr && ApiSpecificConfig::getBindlessMode(device); + + const bool useGlobalImageDescriptors = bindlessHeapsHelper != nullptr && (useGlobalHeaps || dispatchFlags.useBindlessImages); auto &hwInfo = this->peekHwInfo(); @@ -1698,7 +1701,7 @@ inline void CommandStreamReceiverHw::programStateBaseAddress(const In size_t dynamicStateSize = 0; if (hasDsh) { dynamicStateBaseAddress = NEO::getStateBaseAddress(*dsh, useGlobalHeaps); - dynamicStateSize = NEO::getStateSize(*dsh, bindlessHeapsHelper); + dynamicStateSize = NEO::getStateSize(*dsh, useGlobalHeaps); } int64_t surfaceStateBaseAddress = 0; @@ -1734,7 +1737,9 @@ inline void CommandStreamReceiverHw::programStateBaseAddress(const In surfaceStateBaseAddress, surfaceStateSize); } - bool isStateBaseAddressDirty = dshDirty || iohDirty || sshDirty || stateBaseAddressDirty; + const bool imageHeapSelectionChanged = useGlobalImageDescriptors != lastUseGlobalImageDescriptors; + bool isStateBaseAddressDirty = dshDirty || iohDirty || sshDirty || stateBaseAddressDirty || imageHeapSelectionChanged; + handleStateBaseAddressStateTransition(dispatchFlags, isStateBaseAddressDirty); // reprogram state base address command if required @@ -1785,7 +1790,8 @@ inline void CommandStreamReceiverHw::reprogramStateBaseAddress(const commandStreamCSR, bindingTableBaseAddressRequired, dispatchFlags.areMultipleSubDevicesInContext, - true); + true, + dispatchFlags.useBindlessImages); bindingTableBaseAddressRequired = false; setGSBAStateDirty(false); @@ -1805,7 +1811,8 @@ inline void CommandStreamReceiverHw::programStateBaseAddressCommon( LinearStream &csrCommandStream, bool dispatchBindingTableCommand, bool areMultipleSubDevicesInContext, - bool setGeneralStateBaseAddress) { + bool setGeneralStateBaseAddress, + bool useBindlessImages) { using STATE_BASE_ADDRESS = typename GfxFamily::STATE_BASE_ADDRESS; auto &rootDeviceEnvironment = this->peekRootDeviceEnvironment(); @@ -1820,12 +1827,22 @@ inline void CommandStreamReceiverHw::programStateBaseAddressCommon( auto globalHeapsBase = 0ull; bool useGlobalSshAndDsh = false; - if (device.getBindlessHeapsHelper()) { - bindlessSurfStateBase = device.getBindlessHeapsHelper()->getGlobalHeapsBase(); - globalHeapsBase = device.getBindlessHeapsHelper()->getGlobalHeapsBase(); + + auto *helper = device.getBindlessHeapsHelper(); + + const bool useGlobalHeaps = helper != nullptr && ApiSpecificConfig::getBindlessMode(device); + + const bool useGlobalImageDescriptors = helper != nullptr && (useGlobalHeaps || useBindlessImages); + + if (useGlobalHeaps) { + globalHeapsBase = helper->getGlobalHeapsBase(); useGlobalSshAndDsh = true; } + if (useGlobalImageDescriptors) { + bindlessSurfStateBase = helper->getGlobalHeapsBase(); + } + STATE_BASE_ADDRESS stateBaseAddressCmd; StateBaseAddressHelperArgs args = { generalStateBaseAddress, // generalStateBaseAddress @@ -1857,6 +1874,8 @@ inline void CommandStreamReceiverHw::programStateBaseAddressCommon( StateBaseAddressHelper::programStateBaseAddressIntoCommandStream(args, csrCommandStream); + lastUseGlobalImageDescriptors = useGlobalImageDescriptors; + bool sbaTrackingEnabled = debuggingEnabled; if (sbaTrackingEnabled) { device.getL0Debugger()->programSbaAddressLoad(csrCommandStream, @@ -2181,7 +2200,7 @@ void CommandStreamReceiverHw::dispatchImmediateFlushStateBaseAddressC if (flushData.stateBaseAddressDirty) { bool btCommandNeeded = this->streamProperties.stateBaseAddress.bindingTablePoolBaseAddress.value != StreamProperty64::initValue; programStateBaseAddressCommon(nullptr, nullptr, nullptr, &this->streamProperties.stateBaseAddress, - 0, 0, flushData.pipelineSelectArgs, device, csrStream, btCommandNeeded, device.getNumGenericSubDevices() > 1, false); + 0, 0, flushData.pipelineSelectArgs, device, csrStream, btCommandNeeded, device.getNumGenericSubDevices() > 1, false,false); this->streamProperties.stateBaseAddress.clearIsDirty(); } } diff --git a/shared/source/command_stream/csr_definitions.h b/shared/source/command_stream/csr_definitions.h index 5a2b141786d1b..39d93e96d0ac0 100644 --- a/shared/source/command_stream/csr_definitions.h +++ b/shared/source/command_stream/csr_definitions.h @@ -126,6 +126,7 @@ struct DispatchFlags { bool isStallingCommandsOnNextFlushRequired = false; bool isDcFlushRequiredOnStallingCommandsOnNextFlush = false; bool isWalkerWithProfilingEnqueued = false; + bool useBindlessImages = false; }; struct CsrSizeRequestFlags { diff --git a/shared/source/device/root_device.cpp b/shared/source/device/root_device.cpp index d21ab286575c1..d97b3a16bdf22 100644 --- a/shared/source/device/root_device.cpp +++ b/shared/source/device/root_device.cpp @@ -51,8 +51,10 @@ Device *RootDevice::getRootDevice() const { void RootDevice::createBindlessHeapsHelper() { -if (!NEO::debugManager.flags.NEO_L0_SYSMAN_NO_CONTEXT_MODE.get() && ApiSpecificConfig::getGlobalBindlessHeapConfiguration(this->getHardwareInfo()) && ApiSpecificConfig::getBindlessMode(*this)) { - this->executionEnvironment->rootDeviceEnvironments[getRootDeviceIndex()]->createBindlessHeapsHelper(this, getNumGenericSubDevices() > 1); + if (!NEO::debugManager.flags.NEO_L0_SYSMAN_NO_CONTEXT_MODE.get() && + ApiSpecificConfig::createBindlessHeapsHelper(*this)) { + this->executionEnvironment->rootDeviceEnvironments[getRootDeviceIndex()]->createBindlessHeapsHelper( + this, getNumGenericSubDevices() > 1); } } diff --git a/shared/source/execution_environment/root_device_environment.cpp b/shared/source/execution_environment/root_device_environment.cpp index 37743931457f8..2df7b725ec138 100644 --- a/shared/source/execution_environment/root_device_environment.cpp +++ b/shared/source/execution_environment/root_device_environment.cpp @@ -159,7 +159,11 @@ const ProductHelper &RootDeviceEnvironment::getProductHelper() const { } void RootDeviceEnvironment::createBindlessHeapsHelper(Device *rootDevice, bool availableDevices) { - bindlessHeapsHelper = std::make_unique(rootDevice, availableDevices); + std::lock_guard lock(bindlessHeapsHelperMutex); + + if (bindlessHeapsHelper == nullptr) { + bindlessHeapsHelper = std::make_unique(rootDevice, availableDevices); + } } FrontEndController *RootDeviceEnvironment::getFrontEndController() const { diff --git a/shared/source/execution_environment/root_device_environment.h b/shared/source/execution_environment/root_device_environment.h index e3915c5dae4d0..e9e8052f70c40 100644 --- a/shared/source/execution_environment/root_device_environment.h +++ b/shared/source/execution_environment/root_device_environment.h @@ -163,6 +163,7 @@ struct RootDeviceEnvironment : NonCopyableClass { private: std::mutex mtx; + std::mutex bindlessHeapsHelperMutex; }; static_assert(NEO::NonCopyable); diff --git a/shared/source/gmm_helper/cache_settings_helper.cpp b/shared/source/gmm_helper/cache_settings_helper.cpp index b605199d161cd..505af247d73c7 100644 --- a/shared/source/gmm_helper/cache_settings_helper.cpp +++ b/shared/source/gmm_helper/cache_settings_helper.cpp @@ -93,6 +93,7 @@ GmmResourceUsageType CacheSettingsHelper::getDefaultUsageTypeWithCachingEnabled( return GMM_RESOURCE_USAGE_OCL_IMAGE; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: if (debugManager.flags.DisableCachingForHeaps.get()) { return getDefaultUsageTypeWithCachingDisabled(allocationType, productHelper); } @@ -141,6 +142,7 @@ GmmResourceUsageType CacheSettingsHelper::getDefaultUsageTypeWithCachingDisabled return GMM_RESOURCE_USAGE_OCL_BUFFER_CSR_UC; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: return GMM_RESOURCE_USAGE_OCL_SYSTEM_MEMORY_BUFFER_CACHELINE_MISALIGNED; default: return productHelper.isNewCoherencyModelSupported() ? GMM_RESOURCE_USAGE_OCL_BUFFER_CSR_UC : GMM_RESOURCE_USAGE_OCL_BUFFER_CACHELINE_MISALIGNED; diff --git a/shared/source/helpers/api_specific_config.h b/shared/source/helpers/api_specific_config.h index 7441e077bf375..9f7ed3a4de792 100644 --- a/shared/source/helpers/api_specific_config.h +++ b/shared/source/helpers/api_specific_config.h @@ -44,5 +44,6 @@ struct ApiSpecificConfig { static std::string compilerCacheFileExtension(); static int64_t compilerCacheDefaultEnabled(); static bool isUpdateTagFromWaitEnabledForHeapless(); + static bool createBindlessHeapsHelper(const Device &device); }; } // namespace NEO diff --git a/shared/source/helpers/app_resource_classification.h b/shared/source/helpers/app_resource_classification.h index b529050c69c1e..bdb91fb6c1347 100644 --- a/shared/source/helpers/app_resource_classification.h +++ b/shared/source/helpers/app_resource_classification.h @@ -51,6 +51,7 @@ class AppResourceClassification { case AllocationType::kernelArgsBuffer: case AllocationType::kernelIsaInternal: case AllocationType::linearStream: + case AllocationType::bindlessHeap: case AllocationType::mcs: case AllocationType::preemption: case AllocationType::profilingTagBuffer: diff --git a/shared/source/helpers/app_resource_helper.cpp b/shared/source/helpers/app_resource_helper.cpp index 022107821ecef..011ddd78f4d33 100644 --- a/shared/source/helpers/app_resource_helper.cpp +++ b/shared/source/helpers/app_resource_helper.cpp @@ -56,6 +56,8 @@ const char *AppResourceHelper::getResourceTagStr(AllocationType type) { return "KRLISAIN"; case AllocationType::linearStream: return "LINRSTRM"; + case AllocationType::bindlessHeap: + return "BNDLHEAP"; case AllocationType::mapAllocation: return "MAPALLOC"; case AllocationType::mcs: diff --git a/shared/source/helpers/bindless_heaps_helper.cpp b/shared/source/helpers/bindless_heaps_helper.cpp index eac583390801e..677513fb92eff 100644 --- a/shared/source/helpers/bindless_heaps_helper.cpp +++ b/shared/source/helpers/bindless_heaps_helper.cpp @@ -11,6 +11,7 @@ #include "shared/source/execution_environment/execution_environment.h" #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/gmm_helper/gmm_helper.h" +#include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/compiler_product_helper.h" #include "shared/source/helpers/driver_model_type.h" #include "shared/source/helpers/gfx_core_helper.h" @@ -65,13 +66,14 @@ BindlessHeapsHelper::BindlessHeapsHelper(Device *rootDevice, bool isMultiOsConte } } - auto heapAllocation = getHeapAllocation(size, MemoryConstants::pageSize64k, heapType == BindlesHeapType::specialSsh); + auto heapAllocation = getHeapAllocation(size, MemoryConstants::pageSize64k, heapType == BindlesHeapType::specialSsh, + static_cast(heapType)); UNRECOVERABLE_IF(heapAllocation == nullptr); ssHeapsAllocations.push_back(heapAllocation); surfaceStateHeaps[heapType] = std::make_unique(heapAllocation, true); } - borderColorStates = getHeapAllocation(MemoryConstants::pageSize, MemoryConstants::pageSize, false); + borderColorStates = getHeapAllocation(MemoryConstants::pageSize, MemoryConstants::pageSize, false, BindlesHeapType::specialSsh); UNRECOVERABLE_IF(borderColorStates == nullptr); float borderColorDefault[4] = {0, 0, 0, 0}; memcpy_s(borderColorStates->getUnderlyingBuffer(), sizeof(borderColorDefault), borderColorDefault, sizeof(borderColorDefault)); @@ -105,6 +107,15 @@ bool BindlessHeapsHelper::tryReservingMemoryForSpecialSsh(const size_t size, siz return reservedRange.has_value(); } +AllocationType BindlessHeapsHelper::getHeapAllocationType(BindlesHeapType heapType) const { + if (ApiSpecificConfig::getApiType() == ApiSpecificConfig::OCL && + heapType == BindlesHeapType::globalSsh) { + return AllocationType::bindlessHeap; + } + + return AllocationType::linearStream; +} + bool BindlessHeapsHelper::initializeReservedMemory() { if (reservedMemoryInitialized) { return true; @@ -125,8 +136,8 @@ bool BindlessHeapsHelper::initializeReservedMemory() { heapFrontWindow = std::make_unique(reservedRangeBase, heapFrontWindowSize, MemoryConstants::pageSize64k, 0); heapRegular = std::make_unique(reservedRangeBase + heapFrontWindowSize, heapRegularSize, MemoryConstants::pageSize64k, 0); - memManager->addCustomHeapAllocatorConfig(AllocationType::linearStream, true, rootDeviceIndex, {heapFrontWindow.get(), reservedRangeBase}); - memManager->addCustomHeapAllocatorConfig(AllocationType::linearStream, false, rootDeviceIndex, {heapRegular.get(), reservedRangeBase}); + memManager->addCustomHeapAllocatorConfig(getHeapAllocationType(BindlesHeapType::specialSsh), true, rootDeviceIndex, {heapFrontWindow.get(), reservedRangeBase}); + memManager->addCustomHeapAllocatorConfig(getHeapAllocationType(BindlesHeapType::specialSsh), false, rootDeviceIndex, {heapRegular.get(), reservedRangeBase}); reservedMemoryInitialized = true; return true; @@ -145,13 +156,13 @@ BindlessHeapsHelper::~BindlessHeapsHelper() { reservedRanges.clear(); if (reservedMemoryInitialized) { - memManager->removeCustomHeapAllocatorConfig(AllocationType::linearStream, true, rootDeviceIndex); - memManager->removeCustomHeapAllocatorConfig(AllocationType::linearStream, false, rootDeviceIndex); + memManager->removeCustomHeapAllocatorConfig(getHeapAllocationType(BindlesHeapType::specialSsh), true, rootDeviceIndex); + memManager->removeCustomHeapAllocatorConfig(getHeapAllocationType(BindlesHeapType::specialSsh), false, rootDeviceIndex); } } -GraphicsAllocation *BindlessHeapsHelper::getHeapAllocation(size_t heapSize, size_t alignment, bool allocInFrontWindow) { - auto allocationType = AllocationType::linearStream; +GraphicsAllocation *BindlessHeapsHelper::getHeapAllocation(size_t heapSize, size_t alignment, bool allocInFrontWindow, BindlesHeapType heapType) { + auto allocationType = getHeapAllocationType(heapType); NEO::AllocationProperties properties{rootDeviceIndex, true, heapSize, allocationType, isMultiOsContextCapable, deviceBitfield}; properties.flags.use32BitFrontWindow = allocInFrontWindow; properties.alignment = alignment; @@ -278,7 +289,7 @@ IndirectHeap *BindlessHeapsHelper::getHeap(BindlesHeapType heapType) { bool BindlessHeapsHelper::growHeap(BindlesHeapType heapType) { auto heap = surfaceStateHeaps[heapType].get(); auto allocInFrontWindow = false; - auto newAlloc = getHeapAllocation(globalSshAllocationSize, MemoryConstants::pageSize64k, allocInFrontWindow); + auto newAlloc = getHeapAllocation(globalSshAllocationSize, MemoryConstants::pageSize64k, allocInFrontWindow, heapType); DEBUG_BREAK_IF(newAlloc == nullptr); if (newAlloc == nullptr) { return false; diff --git a/shared/source/helpers/bindless_heaps_helper.h b/shared/source/helpers/bindless_heaps_helper.h index 0eb806c60d28b..1ed9b6f67e173 100644 --- a/shared/source/helpers/bindless_heaps_helper.h +++ b/shared/source/helpers/bindless_heaps_helper.h @@ -44,7 +44,7 @@ class BindlessHeapsHelper : NEO::NonCopyableAndNonMovableClass { BindlessHeapsHelper(Device *rootDevice, bool isMultiOsContextCapable); MOCKABLE_VIRTUAL ~BindlessHeapsHelper(); - GraphicsAllocation *getHeapAllocation(size_t heapSize, size_t alignment, bool allocInFrontWindow); + GraphicsAllocation *getHeapAllocation(size_t heapSize, size_t alignment, bool allocInFrontWindow, BindlesHeapType heapType); MOCKABLE_VIRTUAL SurfaceStateInHeapInfo allocateSSInHeap(size_t ssSize, GraphicsAllocation *surfaceAllocation, BindlesHeapType heapType); uint64_t getGlobalHeapsBase(); @@ -74,6 +74,7 @@ class BindlessHeapsHelper : NEO::NonCopyableAndNonMovableClass { bool initializeReservedMemory(); bool isReservedMemoryModeAvailable(); bool tryReservingMemoryForSpecialSsh(const size_t size, size_t alignment); + AllocationType getHeapAllocationType(BindlesHeapType heapType) const; std::optional reserveMemoryRange(size_t size, size_t alignment, HeapIndex heapIndex); std::mutex mtx; diff --git a/shared/source/helpers/heap_assigner.cpp b/shared/source/helpers/heap_assigner.cpp index 847d1ea8a06dd..23dbbaa55ecdf 100644 --- a/shared/source/helpers/heap_assigner.cpp +++ b/shared/source/helpers/heap_assigner.cpp @@ -30,6 +30,9 @@ HeapIndex HeapAssigner::get32BitHeapIndex(AllocationType allocType, bool useLoca return useFrontWindow ? mapExternalWindowIndex(MemoryManager::selectExternalHeap(useLocalMem)) : MemoryManager::selectExternalHeap(useLocalMem); } bool HeapAssigner::useExternal32BitHeap(AllocationType allocType) { + if (allocType == AllocationType::bindlessHeap) { + return true; + } if (apiAllowExternalHeapForSshAndDsh) { return allocType == AllocationType::linearStream; } diff --git a/shared/source/memory_manager/allocation_type.h b/shared/source/memory_manager/allocation_type.h index 0587ce35fd472..f68099a2cb2e4 100644 --- a/shared/source/memory_manager/allocation_type.h +++ b/shared/source/memory_manager/allocation_type.h @@ -28,6 +28,7 @@ enum class AllocationType { kernelIsa, kernelIsaInternal, linearStream, + bindlessHeap, mapAllocation, mcs, preemption, diff --git a/shared/source/memory_manager/definitions/storage_info.cpp b/shared/source/memory_manager/definitions/storage_info.cpp index ee31c4a26c178..12c35ef613182 100644 --- a/shared/source/memory_manager/definitions/storage_info.cpp +++ b/shared/source/memory_manager/definitions/storage_info.cpp @@ -97,6 +97,7 @@ StorageInfo MemoryManager::createStorageInfoFromProperties(const AllocationPrope case AllocationType::commandBuffer: case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: case AllocationType::syncBuffer: storageInfo.cloningOfPageTables = properties.flags.multiOsContextCapable; if (!properties.flags.multiOsContextCapable) { @@ -232,6 +233,7 @@ DeviceBitfield MemoryManager::computeStorageInfoMemoryBanks(const AllocationProp case AllocationType::commandBuffer: case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: memoryBanks = preferredBank; break; case AllocationType::scratchSurface: diff --git a/shared/source/memory_manager/graphics_allocation.h b/shared/source/memory_manager/graphics_allocation.h index 9aef815cafbc9..d67b7a1bba6bd 100644 --- a/shared/source/memory_manager/graphics_allocation.h +++ b/shared/source/memory_manager/graphics_allocation.h @@ -443,6 +443,7 @@ class GraphicsAllocation : public IDNode, NEO::NonCopyableAn case AllocationType::kernelIsa: case AllocationType::kernelIsaInternal: case AllocationType::linearStream: + case AllocationType::bindlessHeap: case AllocationType::ringBuffer: return false; default: diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp index 089005b7d2938..5e62ee94832f2 100644 --- a/shared/source/memory_manager/memory_manager.cpp +++ b/shared/source/memory_manager/memory_manager.cpp @@ -663,6 +663,7 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo case AllocationType::kernelIsa: case AllocationType::kernelIsaInternal: case AllocationType::linearStream: + case AllocationType::bindlessHeap: case AllocationType::mcs: case AllocationType::preemption: case AllocationType::scratchSurface: @@ -1305,6 +1306,7 @@ bool MemoryManager::isAllocationTypeToCapture(AllocationType type) const { case AllocationType::scratchSurface: case AllocationType::privateSurface: case AllocationType::linearStream: + case AllocationType::bindlessHeap: case AllocationType::internalHeap: return true; default: diff --git a/shared/source/utilities/logger.cpp b/shared/source/utilities/logger.cpp index a775417f553ed..9bad52e64c3e1 100644 --- a/shared/source/utilities/logger.cpp +++ b/shared/source/utilities/logger.cpp @@ -282,6 +282,8 @@ const char *getAllocationTypeString(GraphicsAllocation const *graphicsAllocation return "KERNEL_ISA_INTERNAL"; case AllocationType::linearStream: return "LINEAR_STREAM"; + case AllocationType::bindlessHeap: + return "BINDLESS_HEAP"; case AllocationType::mapAllocation: return "MAP_ALLOCATION"; case AllocationType::mcs: diff --git a/shared/test/unit_test/api_specific_config_ult.cpp b/shared/test/unit_test/api_specific_config_ult.cpp index 15fb234eed7c3..587947f638de1 100644 --- a/shared/test/unit_test/api_specific_config_ult.cpp +++ b/shared/test/unit_test/api_specific_config_ult.cpp @@ -7,7 +7,7 @@ #include "shared/source/debug_settings/debug_settings_manager.h" #include "shared/source/helpers/api_specific_config.h" - +#include "shared/source/device/device.h" #include #include @@ -130,5 +130,9 @@ bool ApiSpecificConfig::isGlobalStatelessEnabled(const RootDeviceEnvironment &ro return globalStatelessOcl; } } +bool ApiSpecificConfig::createBindlessHeapsHelper(const Device &device) { + return getGlobalBindlessHeapConfiguration(device.getHardwareInfo()) && + getBindlessMode(device); +} } // namespace NEO diff --git a/shared/test/unit_test/gmm_helper/gmm_helper_tests.cpp b/shared/test/unit_test/gmm_helper/gmm_helper_tests.cpp index 40b7425088efe..a925fd6d58f47 100644 --- a/shared/test/unit_test/gmm_helper/gmm_helper_tests.cpp +++ b/shared/test/unit_test/gmm_helper/gmm_helper_tests.cpp @@ -769,6 +769,7 @@ TEST(GmmTest, givenAllocationTypeWhenGettingUsageTypeThenReturnCorrectValue) { break; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: expectedUsage = forceUncached ? GMM_RESOURCE_USAGE_OCL_SYSTEM_MEMORY_BUFFER_CACHELINE_MISALIGNED : GMM_RESOURCE_USAGE_OCL_STATE_HEAP_BUFFER; break; case AllocationType::gpuTimestampDeviceBuffer: @@ -818,6 +819,7 @@ TEST(GmmTest, givenAllocationTypeWhenGettingUsageTypeThenReturnCorrectValue) { break; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: expectedUsage = forceUncached ? GMM_RESOURCE_USAGE_OCL_SYSTEM_MEMORY_BUFFER_CACHELINE_MISALIGNED : GMM_RESOURCE_USAGE_OCL_STATE_HEAP_BUFFER; break; case AllocationType::gpuTimestampDeviceBuffer: @@ -888,6 +890,7 @@ TEST(GmmTest, givenAllocationTypeAndMitigatedDcFlushWhenGettingUsageTypeThenRetu break; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: expectedUsage = GMM_RESOURCE_USAGE_OCL_STATE_HEAP_BUFFER; break; case AllocationType::bufferHostMemory: @@ -926,6 +929,7 @@ TEST(GmmTest, givenAllocationTypeAndMitigatedDcFlushWhenGettingUsageTypeThenRetu break; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: expectedUsage = GMM_RESOURCE_USAGE_OCL_STATE_HEAP_BUFFER; break; case AllocationType::bufferHostMemory: @@ -1001,7 +1005,9 @@ TEST(GmmTest, givenForceAllResourcesUncachedFlagSetWhenGettingUsageTypeThenRetur if (allocationType == AllocationType::preemption) { expectedUsage = GMM_RESOURCE_USAGE_OCL_BUFFER_CSR_UC; - } else if ((allocationType == AllocationType::internalHeap) || (allocationType == AllocationType::linearStream)) { + } else if ((allocationType == AllocationType::internalHeap) || + (allocationType == AllocationType::linearStream) || + (allocationType == AllocationType::bindlessHeap)) { expectedUsage = GMM_RESOURCE_USAGE_OCL_SYSTEM_MEMORY_BUFFER_CACHELINE_MISALIGNED; } diff --git a/shared/test/unit_test/helpers/app_resource_classification_tests.cpp b/shared/test/unit_test/helpers/app_resource_classification_tests.cpp index 2f82b2830b980..7d7d0809f5a58 100644 --- a/shared/test/unit_test/helpers/app_resource_classification_tests.cpp +++ b/shared/test/unit_test/helpers/app_resource_classification_tests.cpp @@ -44,6 +44,7 @@ TEST(AppResourceClassificationTest, givenInternalAllocationTypesWhenCheckingIsAp EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::kernelArgsBuffer)); EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::kernelIsaInternal)); EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::linearStream)); + EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::bindlessHeap)); EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::mcs)); EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::preemption)); EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::profilingTagBuffer)); @@ -68,7 +69,7 @@ TEST(AppResourceClassificationTest, givenInternalAllocationTypesWhenCheckingIsAp TEST(AppResourceClassificationTest, givenAllAllocationTypesWhenCheckingIsApplicationResourceThenAllTypesAreCovered) { constexpr int expectedApplicationTypes = 20; - constexpr int expectedDriverTypes = 29; + constexpr int expectedDriverTypes = 30; int applicationTypesCount = 0; int driverTypesCount = 0; diff --git a/shared/test/unit_test/helpers/app_resource_tests.cpp b/shared/test/unit_test/helpers/app_resource_tests.cpp index 78ecbb3746f8a..db17312f34cfc 100644 --- a/shared/test/unit_test/helpers/app_resource_tests.cpp +++ b/shared/test/unit_test/helpers/app_resource_tests.cpp @@ -71,6 +71,7 @@ AllocationTypeTagTestCase allocationTypeTagValues[static_cast(AllocationTyp {AllocationType::kernelIsa, "KERNLISA"}, {AllocationType::kernelIsaInternal, "KRLISAIN"}, {AllocationType::linearStream, "LINRSTRM"}, + {AllocationType::bindlessHeap, "BNDLHEAP"}, {AllocationType::mapAllocation, "MAPALLOC"}, {AllocationType::mcs, "MCS"}, {AllocationType::preemption, "PRMPTION"}, diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_bindless_heap_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_bindless_heap_tests.cpp index 5a64549d360a4..d7e766a3d85ce 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_bindless_heap_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_bindless_heap_tests.cpp @@ -7,6 +7,7 @@ #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/gmm_helper/gmm_helper.h" +#include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/bindless_heaps_helper.h" #include "shared/source/indirect_heap/indirect_heap.h" #include "shared/source/os_interface/device_factory.h" @@ -20,16 +21,22 @@ #include "shared/test/common/test_macros/hw_test.h" namespace NEO { +extern ApiSpecificConfig::ApiType apiTypeForUlts; + struct GlobalBindlessDrmMemManagerFixture : public DrmMemoryManagerFixtureWithoutQuietIoctlExpectation { GlobalBindlessDrmMemManagerFixture() : DrmMemoryManagerFixtureWithoutQuietIoctlExpectation(1, 0) {} void setUp() { + previousApiType = apiTypeForUlts; + apiTypeForUlts = ApiSpecificConfig::L0; debugManager.flags.UseExternalAllocatorForSshAndDsh.set(true); DrmMemoryManagerFixtureWithoutQuietIoctlExpectation::setUp(true); } void tearDown() { + apiTypeForUlts = previousApiType; } DebugManagerStateRestore dbgRestorer; + ApiSpecificConfig::ApiType previousApiType = ApiSpecificConfig::OCL; }; using DrmGlobalBindlessAllocatorTests = Test;