356 lines
12 KiB
C++
356 lines
12 KiB
C++
/*
|
|
* Copyright (C) 2018-2023 Intel Corporation
|
|
*
|
|
* SPDX-License-Identifier: MIT
|
|
*
|
|
*/
|
|
|
|
#pragma once
|
|
#include "shared/source/gmm_helper/gmm_lib.h"
|
|
#include "shared/source/helpers/driver_model_type.h"
|
|
#include "shared/source/memory_manager/definitions/engine_limits.h"
|
|
#include "shared/source/os_interface/linux/drm_debug.h"
|
|
#include "shared/source/os_interface/linux/drm_wrappers.h"
|
|
#include "shared/source/os_interface/linux/hw_device_id.h"
|
|
#include "shared/source/os_interface/os_interface.h"
|
|
#include "shared/source/utilities/stackvec.h"
|
|
|
|
#include "igfxfmid.h"
|
|
|
|
#include <array>
|
|
#include <cstdint>
|
|
#include <limits>
|
|
#include <memory>
|
|
#include <mutex>
|
|
#include <string>
|
|
#include <unordered_map>
|
|
#include <vector>
|
|
|
|
struct GT_SYSTEM_INFO;
|
|
|
|
namespace aub_stream {
|
|
enum EngineType : uint32_t;
|
|
}
|
|
|
|
namespace NEO {
|
|
constexpr uint32_t contextPrivateParamBoost = 0x80000000;
|
|
|
|
enum class AllocationType;
|
|
enum class CachePolicy : uint32_t;
|
|
enum class CacheRegion : uint16_t;
|
|
enum class SubmissionStatus : uint32_t;
|
|
|
|
class BufferObject;
|
|
class CompilerProductHelper;
|
|
class DeviceFactory;
|
|
class MemoryInfo;
|
|
class OsContext;
|
|
class OsContextLinux;
|
|
class Gmm;
|
|
struct CacheInfo;
|
|
struct EngineInfo;
|
|
struct HardwareInfo;
|
|
struct RootDeviceEnvironment;
|
|
struct SystemInfo;
|
|
|
|
struct DeviceDescriptor {
|
|
unsigned short deviceId;
|
|
const HardwareInfo *pHwInfo;
|
|
void (*setupHardwareInfo)(HardwareInfo *, bool, const CompilerProductHelper &);
|
|
const char *devName;
|
|
};
|
|
|
|
extern const DeviceDescriptor deviceDescriptorTable[];
|
|
|
|
class Drm : public DriverModel {
|
|
friend DeviceFactory;
|
|
|
|
public:
|
|
static constexpr DriverModelType driverModelType = DriverModelType::DRM;
|
|
|
|
static SubmissionStatus getSubmissionStatusFromReturnCode(int32_t retCode);
|
|
|
|
struct QueryTopologyData {
|
|
int sliceCount;
|
|
int subSliceCount;
|
|
int euCount;
|
|
|
|
int maxSliceCount;
|
|
int maxSubSliceCount;
|
|
int maxEuCount;
|
|
};
|
|
|
|
~Drm() override;
|
|
|
|
virtual int ioctl(DrmIoctl request, void *arg);
|
|
|
|
unsigned int getDeviceHandle() const override {
|
|
return 0;
|
|
}
|
|
PhysicalDevicePciSpeedInfo getPciSpeedInfo() const override;
|
|
int getExecSoftPin(int &execSoftPin);
|
|
int enableTurboBoost();
|
|
int getEuTotal(int &euTotal);
|
|
int getSubsliceTotal(int &subsliceTotal);
|
|
|
|
int getMaxGpuFrequency(HardwareInfo &hwInfo, int &maxGpuFrequency);
|
|
int getEnabledPooledEu(int &enabled);
|
|
int getMinEuInPool(int &minEUinPool);
|
|
int getTimestampFrequency(int &frequency);
|
|
|
|
MOCKABLE_VIRTUAL int queryGttSize(uint64_t >tSizeOutput);
|
|
bool isPreemptionSupported() const { return preemptionSupported; }
|
|
|
|
MOCKABLE_VIRTUAL void checkPreemptionSupport();
|
|
inline int getFileDescriptor() const { return hwDeviceId->getFileDescriptor(); }
|
|
int queryAdapterBDF();
|
|
MOCKABLE_VIRTUAL int createDrmVirtualMemory(uint32_t &drmVmId);
|
|
void destroyDrmVirtualMemory(uint32_t drmVmId);
|
|
MOCKABLE_VIRTUAL int createDrmContext(uint32_t drmVmId, bool isDirectSubmissionRequested, bool isCooperativeContextRequested);
|
|
void destroyDrmContext(uint32_t drmContextId);
|
|
int queryVmId(uint32_t drmContextId, uint32_t &vmId);
|
|
void setLowPriorityContextParam(uint32_t drmContextId);
|
|
|
|
unsigned int bindDrmContext(uint32_t drmContextId, uint32_t deviceIndex, aub_stream::EngineType engineType, bool engineInstancedDevice);
|
|
|
|
MOCKABLE_VIRTUAL int getErrno();
|
|
bool setQueueSliceCount(uint64_t sliceCount);
|
|
void checkQueueSliceSupport();
|
|
uint64_t getSliceMask(uint64_t sliceCount);
|
|
MOCKABLE_VIRTUAL bool querySystemInfo();
|
|
MOCKABLE_VIRTUAL bool queryEngineInfo();
|
|
MOCKABLE_VIRTUAL bool sysmanQueryEngineInfo();
|
|
MOCKABLE_VIRTUAL bool queryEngineInfo(bool isSysmanEnabled);
|
|
MOCKABLE_VIRTUAL bool queryMemoryInfo();
|
|
bool queryTopology(const HardwareInfo &hwInfo, QueryTopologyData &data);
|
|
bool createVirtualMemoryAddressSpace(uint32_t vmCount);
|
|
void destroyVirtualMemoryAddressSpace();
|
|
uint32_t getVirtualMemoryAddressSpace(uint32_t vmId) const;
|
|
MOCKABLE_VIRTUAL int bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo);
|
|
MOCKABLE_VIRTUAL int unbindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo);
|
|
int setupHardwareInfo(const DeviceDescriptor *, bool);
|
|
void setupSystemInfo(HardwareInfo *hwInfo, SystemInfo *sysInfo);
|
|
void setupCacheInfo(const HardwareInfo &hwInfo);
|
|
MOCKABLE_VIRTUAL void getPrelimVersion(std::string &prelimVersion);
|
|
MOCKABLE_VIRTUAL void getPrelimEuDebug(int &prelimEuDebug);
|
|
|
|
PhysicalDevicePciBusInfo getPciBusInfo() const override;
|
|
bool isGpuHangDetected(OsContext &osContext) override;
|
|
|
|
bool areNonPersistentContextsSupported() const { return nonPersistentContextsSupported; }
|
|
void checkNonPersistentContextsSupport();
|
|
void setNonPersistentContext(uint32_t drmContextId);
|
|
bool isPerContextVMRequired() const {
|
|
return requirePerContextVM;
|
|
}
|
|
void setPerContextVMRequired(bool required) {
|
|
requirePerContextVM = required;
|
|
}
|
|
|
|
void checkContextDebugSupport();
|
|
bool isContextDebugSupported() { return contextDebugSupported; }
|
|
MOCKABLE_VIRTUAL void setContextDebugFlag(uint32_t drmContextId);
|
|
|
|
void setUnrecoverableContext(uint32_t drmContextId);
|
|
|
|
void setDirectSubmissionActive(bool value) { this->directSubmissionActive = value; }
|
|
bool isDirectSubmissionActive() const { return this->directSubmissionActive; }
|
|
|
|
MOCKABLE_VIRTUAL bool isSetPairAvailable();
|
|
MOCKABLE_VIRTUAL bool getSetPairAvailable() { return setPairAvailable; }
|
|
MOCKABLE_VIRTUAL bool isChunkingAvailable();
|
|
MOCKABLE_VIRTUAL bool getChunkingAvailable() { return chunkingAvailable; }
|
|
|
|
MOCKABLE_VIRTUAL bool useVMBindImmediate() const;
|
|
|
|
MOCKABLE_VIRTUAL bool isVmBindAvailable();
|
|
MOCKABLE_VIRTUAL bool registerResourceClasses();
|
|
|
|
MOCKABLE_VIRTUAL void queryPageFaultSupport();
|
|
bool hasPageFaultSupport() const;
|
|
bool hasKmdMigrationSupport() const;
|
|
|
|
MOCKABLE_VIRTUAL uint32_t registerResource(DrmResourceClass classType, const void *data, size_t size);
|
|
MOCKABLE_VIRTUAL void unregisterResource(uint32_t handle);
|
|
MOCKABLE_VIRTUAL uint32_t registerIsaCookie(uint32_t isaHandle);
|
|
|
|
MOCKABLE_VIRTUAL bool isDebugAttachAvailable();
|
|
|
|
void setGmmInputArgs(void *args) override;
|
|
|
|
SystemInfo *getSystemInfo() const {
|
|
return systemInfo.get();
|
|
}
|
|
|
|
CacheInfo *getCacheInfo() const {
|
|
return cacheInfo.get();
|
|
}
|
|
|
|
MemoryInfo *getMemoryInfo() const {
|
|
return memoryInfo.get();
|
|
}
|
|
|
|
EngineInfo *getEngineInfo() const {
|
|
return engineInfo.get();
|
|
}
|
|
|
|
IoctlHelper *getIoctlHelper() const {
|
|
return ioctlHelper.get();
|
|
}
|
|
|
|
const RootDeviceEnvironment &getRootDeviceEnvironment() const {
|
|
return rootDeviceEnvironment;
|
|
}
|
|
|
|
bool resourceRegistrationEnabled() {
|
|
return classHandles.size() > 0;
|
|
}
|
|
|
|
static bool isDrmSupported(int fileDescriptor);
|
|
|
|
static Drm *create(std::unique_ptr<HwDeviceIdDrm> &&hwDeviceId, RootDeviceEnvironment &rootDeviceEnvironment);
|
|
static void overrideBindSupport(bool &useVmBind);
|
|
std::string getPciPath() {
|
|
return hwDeviceId->getPciPath();
|
|
}
|
|
|
|
void waitForBind(uint32_t vmHandleId);
|
|
uint64_t getNextFenceVal(uint32_t vmHandleId) { return fenceVal[vmHandleId] + 1; }
|
|
void incFenceVal(uint32_t vmHandleId) { fenceVal[vmHandleId]++; }
|
|
uint64_t *getFenceAddr(uint32_t vmHandleId) { return &pagingFence[vmHandleId]; }
|
|
|
|
int waitHandle(uint32_t waitHandle, int64_t timeout);
|
|
enum class ValueWidth : uint32_t {
|
|
U8,
|
|
U16,
|
|
U32,
|
|
U64
|
|
};
|
|
MOCKABLE_VIRTUAL int waitUserFence(uint32_t ctxId, uint64_t address, uint64_t value, ValueWidth dataWidth, int64_t timeout, uint16_t flags);
|
|
|
|
void waitOnUserFences(const OsContextLinux &osContext, uint64_t address, uint64_t value, uint32_t numActiveTiles, uint32_t postSyncOffset);
|
|
|
|
void setNewResourceBoundToVM(BufferObject *bo, uint32_t vmHandleId);
|
|
|
|
const std::vector<int> &getSliceMappings(uint32_t deviceIndex);
|
|
|
|
static std::vector<std::unique_ptr<HwDeviceId>> discoverDevices(ExecutionEnvironment &executionEnvironment);
|
|
static std::vector<std::unique_ptr<HwDeviceId>> discoverDevice(ExecutionEnvironment &executionEnvironment, std::string &osPciPath);
|
|
static std::vector<std::unique_ptr<HwDeviceId>> discoverDevices(ExecutionEnvironment &executionEnvironment, std::string &osPciPath);
|
|
|
|
[[nodiscard]] std::unique_lock<std::mutex> lockBindFenceMutex();
|
|
|
|
void setPciDomain(uint32_t domain) {
|
|
pciDomain = domain;
|
|
}
|
|
|
|
MOCKABLE_VIRTUAL std::vector<uint8_t> getMemoryRegions();
|
|
|
|
MOCKABLE_VIRTUAL bool completionFenceSupport();
|
|
|
|
MOCKABLE_VIRTUAL uint32_t notifyFirstCommandQueueCreated(const void *data, size_t size);
|
|
MOCKABLE_VIRTUAL void notifyLastCommandQueueDestroyed(uint32_t handle);
|
|
|
|
uint64_t getPatIndex(Gmm *gmm, AllocationType allocationType, CacheRegion cacheRegion, CachePolicy cachePolicy, bool closEnabled) const;
|
|
bool isVmBindPatIndexProgrammingSupported() const { return vmBindPatIndexProgrammingSupported; }
|
|
MOCKABLE_VIRTUAL bool getDeviceMemoryMaxClockRateInMhz(uint32_t tileId, uint32_t &clkRate);
|
|
MOCKABLE_VIRTUAL bool getDeviceMemoryPhysicalSizeInBytes(uint32_t tileId, uint64_t &physicalSize);
|
|
void cleanup() override;
|
|
bool readSysFsAsString(const std::string &relativeFilePath, std::string &readString);
|
|
MOCKABLE_VIRTUAL std::string getSysFsPciPath();
|
|
std::unique_ptr<HwDeviceIdDrm> &getHwDeviceId() { return hwDeviceId; }
|
|
std::vector<uint8_t> query(uint32_t queryId, uint32_t queryItemFlags);
|
|
|
|
protected:
|
|
Drm(std::unique_ptr<HwDeviceIdDrm> &&hwDeviceIdIn, RootDeviceEnvironment &rootDeviceEnvironment);
|
|
|
|
int getQueueSliceCount(GemContextParamSseu *sseu);
|
|
bool translateTopologyInfo(const QueryTopologyInfo *queryTopologyInfo, QueryTopologyData &data, TopologyMapping &mapping);
|
|
std::string generateUUID();
|
|
std::string generateElfUUID(const void *data);
|
|
void printIoctlStatistics();
|
|
void setupIoctlHelper(const PRODUCT_FAMILY productFamily);
|
|
void queryAndSetVmBindPatIndexProgrammingSupport();
|
|
static std::string getDrmVersion(int fileDescriptor);
|
|
bool queryDeviceIdAndRevision();
|
|
bool queryI915DeviceIdAndRevision();
|
|
|
|
#pragma pack(1)
|
|
struct PCIConfig {
|
|
uint16_t vendorID;
|
|
uint16_t deviceID;
|
|
uint16_t command;
|
|
uint16_t status;
|
|
uint8_t revision;
|
|
uint8_t progIF;
|
|
uint8_t subclass;
|
|
uint8_t classCode;
|
|
uint8_t cacheLineSize;
|
|
uint8_t latencyTimer;
|
|
uint8_t headerType;
|
|
uint8_t bist;
|
|
uint32_t bar0[6];
|
|
uint32_t cardbusCISPointer;
|
|
uint16_t subsystemVendorID;
|
|
uint16_t subsystemDeviceID;
|
|
uint32_t rom;
|
|
uint8_t capabilities;
|
|
uint8_t reserved[7];
|
|
uint8_t interruptLine;
|
|
uint8_t interruptPIN;
|
|
uint8_t minGrant;
|
|
uint8_t maxLatency;
|
|
};
|
|
#pragma pack()
|
|
|
|
GemContextParamSseu sseu{};
|
|
ADAPTER_BDF adapterBDF{};
|
|
uint32_t pciDomain = 0;
|
|
|
|
struct IoctlStatisticsEntry {
|
|
long long totalTime = 0;
|
|
uint64_t count = 0;
|
|
long long minTime = std::numeric_limits<long long>::max();
|
|
long long maxTime = 0;
|
|
};
|
|
std::unordered_map<DrmIoctl, IoctlStatisticsEntry> ioctlStatistics;
|
|
|
|
std::mutex bindFenceMutex;
|
|
std::array<uint64_t, EngineLimits::maxHandleCount> pagingFence;
|
|
std::array<uint64_t, EngineLimits::maxHandleCount> fenceVal;
|
|
StackVec<uint32_t, size_t(DrmResourceClass::MaxSize)> classHandles;
|
|
std::vector<uint32_t> virtualMemoryIds;
|
|
|
|
std::unique_ptr<HwDeviceIdDrm> hwDeviceId;
|
|
std::unique_ptr<IoctlHelper> ioctlHelper;
|
|
std::unique_ptr<SystemInfo> systemInfo;
|
|
std::unique_ptr<CacheInfo> cacheInfo;
|
|
std::unique_ptr<EngineInfo> engineInfo;
|
|
std::unique_ptr<MemoryInfo> memoryInfo;
|
|
|
|
std::once_flag checkBindOnce;
|
|
std::once_flag checkSetPairOnce;
|
|
std::once_flag checkChunkingOnce;
|
|
std::once_flag checkCompletionFenceOnce;
|
|
|
|
RootDeviceEnvironment &rootDeviceEnvironment;
|
|
uint64_t uuid = 0;
|
|
|
|
bool sliceCountChangeSupported = false;
|
|
bool preemptionSupported = false;
|
|
bool nonPersistentContextsSupported = false;
|
|
bool requirePerContextVM = false;
|
|
bool bindAvailable = false;
|
|
bool directSubmissionActive = false;
|
|
bool setPairAvailable = false;
|
|
bool chunkingAvailable = false;
|
|
bool contextDebugSupported = false;
|
|
bool pageFaultSupported = false;
|
|
bool completionFenceSupported = false;
|
|
bool vmBindPatIndexProgrammingSupported = false;
|
|
|
|
private:
|
|
int getParamIoctl(DrmParam param, int *dstValue);
|
|
};
|
|
} // namespace NEO
|